2024-12-10 19:28:06,906 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-10 19:28:06,921 main DEBUG Took 0.012606 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 19:28:06,922 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 19:28:06,922 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 19:28:06,924 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 19:28:06,925 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,935 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 19:28:06,951 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,954 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,956 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,958 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,959 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,961 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,963 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,964 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,965 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,965 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,966 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,967 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,967 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,968 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,969 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 19:28:06,970 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,971 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 19:28:06,974 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 19:28:06,976 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 19:28:06,978 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 19:28:06,978 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 19:28:06,980 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 19:28:06,980 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 19:28:06,990 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 19:28:06,993 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 19:28:06,995 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 19:28:06,995 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 19:28:06,995 main DEBUG createAppenders(={Console}) 2024-12-10 19:28:06,996 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-10 19:28:06,996 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-10 19:28:06,997 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-10 19:28:06,997 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 19:28:06,997 main DEBUG OutputStream closed 2024-12-10 19:28:06,997 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 19:28:06,998 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 19:28:06,998 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-10 19:28:07,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 19:28:07,081 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 19:28:07,083 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 19:28:07,084 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 19:28:07,085 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 19:28:07,085 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 19:28:07,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 19:28:07,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 19:28:07,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 19:28:07,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 19:28:07,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 19:28:07,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 19:28:07,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 19:28:07,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 19:28:07,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 19:28:07,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 19:28:07,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 19:28:07,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 19:28:07,097 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 19:28:07,098 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-10 19:28:07,099 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 19:28:07,100 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-10T19:28:07,415 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603 2024-12-10 19:28:07,418 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 19:28:07,418 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T19:28:07,427 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-10T19:28:07,434 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-10T19:28:07,460 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T19:28:07,518 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T19:28:07,518 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T19:28:07,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-10T19:28:07,545 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf, deleteOnExit=true 2024-12-10T19:28:07,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-10T19:28:07,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/test.cache.data in system properties and HBase conf 2024-12-10T19:28:07,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.tmp.dir in system properties and HBase conf 2024-12-10T19:28:07,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.log.dir in system properties and HBase conf 2024-12-10T19:28:07,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-10T19:28:07,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-10T19:28:07,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-10T19:28:07,654 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T19:28:07,797 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T19:28:07,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-10T19:28:07,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-10T19:28:07,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-10T19:28:07,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T19:28:07,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-10T19:28:07,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-10T19:28:07,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T19:28:07,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T19:28:07,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-10T19:28:07,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/nfs.dump.dir in system properties and HBase conf 2024-12-10T19:28:07,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/java.io.tmpdir in system properties and HBase conf 2024-12-10T19:28:07,808 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T19:28:07,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-10T19:28:07,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-10T19:28:09,258 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-10T19:28:09,405 INFO [Time-limited test {}] log.Log(170): Logging initialized @3309ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-10T19:28:09,539 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T19:28:09,683 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T19:28:09,792 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T19:28:09,792 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T19:28:09,800 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T19:28:09,882 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T19:28:09,897 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.log.dir/,AVAILABLE} 2024-12-10T19:28:09,899 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T19:28:10,151 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/java.io.tmpdir/jetty-localhost-44361-hadoop-hdfs-3_4_1-tests_jar-_-any-8539498001759347038/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T19:28:10,166 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:44361} 2024-12-10T19:28:10,166 INFO [Time-limited test {}] server.Server(415): Started @4070ms 2024-12-10T19:28:10,806 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T19:28:10,818 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T19:28:10,821 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T19:28:10,821 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T19:28:10,821 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T19:28:10,823 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.log.dir/,AVAILABLE} 2024-12-10T19:28:10,824 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T19:28:10,932 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/java.io.tmpdir/jetty-localhost-40881-hadoop-hdfs-3_4_1-tests_jar-_-any-14676706151030301317/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T19:28:10,933 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:40881} 2024-12-10T19:28:10,933 INFO [Time-limited test {}] server.Server(415): Started @4838ms 2024-12-10T19:28:10,981 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T19:28:11,108 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T19:28:11,113 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T19:28:11,118 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T19:28:11,118 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T19:28:11,118 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T19:28:11,119 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.log.dir/,AVAILABLE} 2024-12-10T19:28:11,120 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T19:28:11,235 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/java.io.tmpdir/jetty-localhost-45193-hadoop-hdfs-3_4_1-tests_jar-_-any-7625026705726541879/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T19:28:11,237 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:45193} 2024-12-10T19:28:11,237 INFO [Time-limited test {}] server.Server(415): Started @5141ms 2024-12-10T19:28:11,240 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T19:28:11,282 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T19:28:11,287 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T19:28:11,289 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T19:28:11,290 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T19:28:11,290 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T19:28:11,291 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.log.dir/,AVAILABLE} 2024-12-10T19:28:11,292 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T19:28:11,431 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/java.io.tmpdir/jetty-localhost-43799-hadoop-hdfs-3_4_1-tests_jar-_-any-14269586198209534470/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T19:28:11,432 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:43799} 2024-12-10T19:28:11,432 INFO [Time-limited test {}] server.Server(415): Started @5336ms 2024-12-10T19:28:11,435 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T19:28:12,300 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data4/current/BP-1608359495-172.17.0.3-1733858888650/current, will proceed with Du for space computation calculation, 2024-12-10T19:28:12,300 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data1/current/BP-1608359495-172.17.0.3-1733858888650/current, will proceed with Du for space computation calculation, 2024-12-10T19:28:12,300 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data3/current/BP-1608359495-172.17.0.3-1733858888650/current, will proceed with Du for space computation calculation, 2024-12-10T19:28:12,300 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data2/current/BP-1608359495-172.17.0.3-1733858888650/current, will proceed with Du for space computation calculation, 2024-12-10T19:28:12,350 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T19:28:12,350 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T19:28:12,381 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data5/current/BP-1608359495-172.17.0.3-1733858888650/current, will proceed with Du for space computation calculation, 2024-12-10T19:28:12,383 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data6/current/BP-1608359495-172.17.0.3-1733858888650/current, will proceed with Du for space computation calculation, 2024-12-10T19:28:12,415 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T19:28:12,417 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc635288764008e82 with lease ID 0xb9489f924d8f3f3c: Processing first storage report for DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d from datanode DatanodeRegistration(127.0.0.1:43797, datanodeUuid=10ffcbc2-356c-4f0d-b302-01e6f5dbcc2b, infoPort=41811, infoSecurePort=0, ipcPort=43625, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650) 2024-12-10T19:28:12,418 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc635288764008e82 with lease ID 0xb9489f924d8f3f3c: from storage DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d node DatanodeRegistration(127.0.0.1:43797, datanodeUuid=10ffcbc2-356c-4f0d-b302-01e6f5dbcc2b, infoPort=41811, infoSecurePort=0, ipcPort=43625, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-10T19:28:12,419 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21e2ed36d002a425 with lease ID 0xb9489f924d8f3f3d: Processing first storage report for DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab from datanode DatanodeRegistration(127.0.0.1:39621, datanodeUuid=134c8558-deb3-477d-8cc5-a583a3334372, infoPort=45539, infoSecurePort=0, ipcPort=46617, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650) 2024-12-10T19:28:12,419 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21e2ed36d002a425 with lease ID 0xb9489f924d8f3f3d: from storage DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab node DatanodeRegistration(127.0.0.1:39621, datanodeUuid=134c8558-deb3-477d-8cc5-a583a3334372, infoPort=45539, infoSecurePort=0, ipcPort=46617, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-10T19:28:12,419 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc635288764008e82 with lease ID 0xb9489f924d8f3f3c: Processing first storage report for DS-71df9b17-cd49-4a62-97c7-e2cc61ee9f9a from datanode DatanodeRegistration(127.0.0.1:43797, datanodeUuid=10ffcbc2-356c-4f0d-b302-01e6f5dbcc2b, infoPort=41811, infoSecurePort=0, ipcPort=43625, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650) 2024-12-10T19:28:12,420 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc635288764008e82 with lease ID 0xb9489f924d8f3f3c: from storage DS-71df9b17-cd49-4a62-97c7-e2cc61ee9f9a node DatanodeRegistration(127.0.0.1:43797, datanodeUuid=10ffcbc2-356c-4f0d-b302-01e6f5dbcc2b, infoPort=41811, infoSecurePort=0, ipcPort=43625, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T19:28:12,420 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21e2ed36d002a425 with lease ID 0xb9489f924d8f3f3d: Processing first storage report for DS-e86cb864-ab83-4831-be5e-a96ede9e7492 from datanode DatanodeRegistration(127.0.0.1:39621, datanodeUuid=134c8558-deb3-477d-8cc5-a583a3334372, infoPort=45539, infoSecurePort=0, ipcPort=46617, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650) 2024-12-10T19:28:12,420 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21e2ed36d002a425 with lease ID 0xb9489f924d8f3f3d: from storage DS-e86cb864-ab83-4831-be5e-a96ede9e7492 node DatanodeRegistration(127.0.0.1:39621, datanodeUuid=134c8558-deb3-477d-8cc5-a583a3334372, infoPort=45539, infoSecurePort=0, ipcPort=46617, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T19:28:12,426 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x87fa40c85561c254 with lease ID 0xb9489f924d8f3f3e: Processing first storage report for DS-7224b066-4b44-401a-8843-7d969e5a27e1 from datanode DatanodeRegistration(127.0.0.1:35009, datanodeUuid=d842e86a-de0e-42d5-8d40-f4a25915d3a3, infoPort=44649, infoSecurePort=0, ipcPort=37991, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650) 2024-12-10T19:28:12,427 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87fa40c85561c254 with lease ID 0xb9489f924d8f3f3e: from storage DS-7224b066-4b44-401a-8843-7d969e5a27e1 node DatanodeRegistration(127.0.0.1:35009, datanodeUuid=d842e86a-de0e-42d5-8d40-f4a25915d3a3, infoPort=44649, infoSecurePort=0, ipcPort=37991, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T19:28:12,427 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x87fa40c85561c254 with lease ID 0xb9489f924d8f3f3e: Processing first storage report for DS-263343cf-aeae-4158-9aeb-e9462a949301 from datanode DatanodeRegistration(127.0.0.1:35009, datanodeUuid=d842e86a-de0e-42d5-8d40-f4a25915d3a3, infoPort=44649, infoSecurePort=0, ipcPort=37991, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650) 2024-12-10T19:28:12,427 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87fa40c85561c254 with lease ID 0xb9489f924d8f3f3e: from storage DS-263343cf-aeae-4158-9aeb-e9462a949301 node DatanodeRegistration(127.0.0.1:35009, datanodeUuid=d842e86a-de0e-42d5-8d40-f4a25915d3a3, infoPort=44649, infoSecurePort=0, ipcPort=37991, storageInfo=lv=-57;cid=testClusterID;nsid=1633453165;c=1733858888650), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T19:28:12,523 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603 2024-12-10T19:28:12,590 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/zookeeper_0, clientPort=53100, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T19:28:12,600 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53100 2024-12-10T19:28:12,614 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:12,617 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:12,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741825_1001 (size=7) 2024-12-10T19:28:12,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741825_1001 (size=7) 2024-12-10T19:28:12,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741825_1001 (size=7) 2024-12-10T19:28:12,883 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36 with version=8 2024-12-10T19:28:12,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/hbase-staging 2024-12-10T19:28:13,202 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e4ad70981cbb:0 server-side Connection retries=45 2024-12-10T19:28:13,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:13,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:13,215 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T19:28:13,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:13,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T19:28:13,380 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T19:28:13,451 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T19:28:13,460 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T19:28:13,464 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T19:28:13,488 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 36610 (auto-detected) 2024-12-10T19:28:13,489 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-10T19:28:13,507 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41471 2024-12-10T19:28:13,533 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41471 connecting to ZooKeeper ensemble=127.0.0.1:53100 2024-12-10T19:28:13,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414710x0, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T19:28:13,630 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41471-0x100114e4b300000 connected 2024-12-10T19:28:13,729 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:13,733 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:13,743 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T19:28:13,748 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36, hbase.cluster.distributed=false 2024-12-10T19:28:13,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T19:28:13,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41471 2024-12-10T19:28:13,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41471 2024-12-10T19:28:13,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41471 2024-12-10T19:28:13,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41471 2024-12-10T19:28:13,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41471 2024-12-10T19:28:13,909 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e4ad70981cbb:0 server-side Connection retries=45 2024-12-10T19:28:13,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:13,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:13,912 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T19:28:13,912 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:13,912 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T19:28:13,915 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T19:28:13,917 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T19:28:13,918 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41557 2024-12-10T19:28:13,919 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41557 connecting to ZooKeeper ensemble=127.0.0.1:53100 2024-12-10T19:28:13,920 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:13,926 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:13,942 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415570x0, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T19:28:13,943 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41557-0x100114e4b300001 connected 2024-12-10T19:28:13,943 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T19:28:13,948 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T19:28:13,958 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T19:28:13,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T19:28:13,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T19:28:13,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41557 2024-12-10T19:28:13,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41557 2024-12-10T19:28:13,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41557 2024-12-10T19:28:13,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41557 2024-12-10T19:28:13,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41557 2024-12-10T19:28:13,985 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e4ad70981cbb:0 server-side Connection retries=45 2024-12-10T19:28:13,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:13,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:13,986 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T19:28:13,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:13,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T19:28:13,987 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T19:28:13,987 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T19:28:13,988 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39803 2024-12-10T19:28:13,990 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39803 connecting to ZooKeeper ensemble=127.0.0.1:53100 2024-12-10T19:28:13,991 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:13,994 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:14,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:398030x0, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T19:28:14,018 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39803-0x100114e4b300002 connected 2024-12-10T19:28:14,019 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T19:28:14,020 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T19:28:14,026 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T19:28:14,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T19:28:14,037 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T19:28:14,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39803 2024-12-10T19:28:14,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39803 2024-12-10T19:28:14,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39803 2024-12-10T19:28:14,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39803 2024-12-10T19:28:14,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39803 2024-12-10T19:28:14,074 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e4ad70981cbb:0 server-side Connection retries=45 2024-12-10T19:28:14,074 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:14,075 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:14,075 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T19:28:14,075 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T19:28:14,075 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T19:28:14,075 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T19:28:14,076 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T19:28:14,085 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35427 2024-12-10T19:28:14,087 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35427 connecting to ZooKeeper ensemble=127.0.0.1:53100 2024-12-10T19:28:14,090 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:14,102 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:14,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354270x0, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T19:28:14,118 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:354270x0, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T19:28:14,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35427-0x100114e4b300003 connected 2024-12-10T19:28:14,119 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T19:28:14,120 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T19:28:14,121 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T19:28:14,123 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T19:28:14,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35427 2024-12-10T19:28:14,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35427 2024-12-10T19:28:14,125 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35427 2024-12-10T19:28:14,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35427 2024-12-10T19:28:14,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35427 2024-12-10T19:28:14,144 DEBUG [M:0;e4ad70981cbb:41471 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e4ad70981cbb:41471 2024-12-10T19:28:14,145 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:14,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T19:28:14,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T19:28:14,160 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T19:28:14,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T19:28:14,161 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:14,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T19:28:14,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:14,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T19:28:14,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:14,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:14,193 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T19:28:14,193 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:14,194 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T19:28:14,197 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e4ad70981cbb,41471,1733858893036 from backup master directory 2024-12-10T19:28:14,209 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T19:28:14,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:14,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T19:28:14,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T19:28:14,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T19:28:14,211 WARN [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T19:28:14,211 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:14,214 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T19:28:14,215 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T19:28:14,279 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/hbase.id] with ID: 896f6e2b-5117-4154-b1cc-9fad4e023b6a 2024-12-10T19:28:14,279 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/.tmp/hbase.id 2024-12-10T19:28:14,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741826_1002 (size=42) 2024-12-10T19:28:14,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741826_1002 (size=42) 2024-12-10T19:28:14,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741826_1002 (size=42) 2024-12-10T19:28:14,306 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/.tmp/hbase.id]:[hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/hbase.id] 2024-12-10T19:28:14,353 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T19:28:14,359 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T19:28:14,383 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-12-10T19:28:14,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:14,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:14,397 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:14,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:14,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741827_1003 (size=196) 2024-12-10T19:28:14,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741827_1003 (size=196) 2024-12-10T19:28:14,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741827_1003 (size=196) 2024-12-10T19:28:14,442 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T19:28:14,444 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T19:28:14,452 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:14,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741828_1004 (size=1189) 2024-12-10T19:28:14,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741828_1004 (size=1189) 2024-12-10T19:28:14,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741828_1004 (size=1189) 2024-12-10T19:28:14,942 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store 2024-12-10T19:28:14,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741829_1005 (size=34) 2024-12-10T19:28:14,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741829_1005 (size=34) 2024-12-10T19:28:14,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741829_1005 (size=34) 2024-12-10T19:28:14,967 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T19:28:14,970 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:14,971 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T19:28:14,971 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T19:28:14,972 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T19:28:14,973 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T19:28:14,973 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T19:28:14,973 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T19:28:14,974 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733858894971Disabling compacts and flushes for region at 1733858894971Disabling writes for close at 1733858894973 (+2 ms)Writing region close event to WAL at 1733858894973Closed at 1733858894973 2024-12-10T19:28:14,976 WARN [master/e4ad70981cbb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/.initializing 2024-12-10T19:28:14,976 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/WALs/e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:14,985 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:15,000 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4ad70981cbb%2C41471%2C1733858893036, suffix=, logDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/WALs/e4ad70981cbb,41471,1733858893036, archiveDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/oldWALs, maxLogs=10 2024-12-10T19:28:15,030 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/WALs/e4ad70981cbb,41471,1733858893036/e4ad70981cbb%2C41471%2C1733858893036.1733858895007, exclude list is [], retry=0 2024-12-10T19:28:15,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:15,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:15,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:15,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:15,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-10T19:28:15,094 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-10T19:28:15,172 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/WALs/e4ad70981cbb,41471,1733858893036/e4ad70981cbb%2C41471%2C1733858893036.1733858895007 2024-12-10T19:28:15,173 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539)] 2024-12-10T19:28:15,174 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:15,175 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:15,179 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,179 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,261 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T19:28:15,267 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:15,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T19:28:15,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T19:28:15,278 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:15,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:15,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,285 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T19:28:15,286 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:15,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:15,289 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,293 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T19:28:15,293 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:15,294 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:15,295 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,299 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,301 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,311 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,312 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,318 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T19:28:15,323 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T19:28:15,330 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:15,332 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67088031, jitterRate=-3.104358911514282E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T19:28:15,341 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733858895194Initializing all the Stores at 1733858895196 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733858895197 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858895198 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858895198Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858895198Cleaning up temporary data from old regions at 1733858895312 (+114 ms)Region opened successfully at 1733858895341 (+29 ms) 2024-12-10T19:28:15,344 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T19:28:15,393 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6af0cbdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e4ad70981cbb/172.17.0.3:0 2024-12-10T19:28:15,433 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T19:28:15,445 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T19:28:15,445 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T19:28:15,449 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T19:28:15,450 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-10T19:28:15,457 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-10T19:28:15,457 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T19:28:15,492 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T19:28:15,502 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T19:28:15,513 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T19:28:15,517 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T19:28:15,520 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T19:28:15,530 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T19:28:15,534 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T19:28:15,541 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T19:28:15,550 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T19:28:15,553 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T19:28:15,563 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T19:28:15,587 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T19:28:15,596 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T19:28:15,609 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T19:28:15,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T19:28:15,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T19:28:15,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T19:28:15,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,610 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,613 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e4ad70981cbb,41471,1733858893036, sessionid=0x100114e4b300000, setting cluster-up flag (Was=false) 2024-12-10T19:28:15,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,642 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,667 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T19:28:15,670 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:15,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,688 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:15,713 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T19:28:15,715 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:15,721 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T19:28:15,736 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(746): ClusterId : 896f6e2b-5117-4154-b1cc-9fad4e023b6a 2024-12-10T19:28:15,738 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(746): ClusterId : 896f6e2b-5117-4154-b1cc-9fad4e023b6a 2024-12-10T19:28:15,739 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(746): ClusterId : 896f6e2b-5117-4154-b1cc-9fad4e023b6a 2024-12-10T19:28:15,740 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T19:28:15,740 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T19:28:15,740 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T19:28:15,753 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T19:28:15,753 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T19:28:15,753 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T19:28:15,753 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T19:28:15,753 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T19:28:15,753 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T19:28:15,781 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T19:28:15,781 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T19:28:15,781 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T19:28:15,782 DEBUG [RS:1;e4ad70981cbb:39803 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ba79eb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e4ad70981cbb/172.17.0.3:0 2024-12-10T19:28:15,782 DEBUG [RS:2;e4ad70981cbb:35427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f2bba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e4ad70981cbb/172.17.0.3:0 2024-12-10T19:28:15,783 DEBUG [RS:0;e4ad70981cbb:41557 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36559f7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e4ad70981cbb/172.17.0.3:0 2024-12-10T19:28:15,799 DEBUG [RS:1;e4ad70981cbb:39803 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e4ad70981cbb:39803 2024-12-10T19:28:15,801 DEBUG [RS:2;e4ad70981cbb:35427 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;e4ad70981cbb:35427 2024-12-10T19:28:15,801 DEBUG [RS:0;e4ad70981cbb:41557 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e4ad70981cbb:41557 2024-12-10T19:28:15,804 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T19:28:15,804 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T19:28:15,804 DEBUG [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T19:28:15,805 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T19:28:15,805 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T19:28:15,806 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T19:28:15,806 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T19:28:15,806 DEBUG [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T19:28:15,806 DEBUG [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T19:28:15,808 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(2659): reportForDuty to master=e4ad70981cbb,41471,1733858893036 with port=35427, startcode=1733858894073 2024-12-10T19:28:15,808 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(2659): reportForDuty to master=e4ad70981cbb,41471,1733858893036 with port=39803, startcode=1733858893985 2024-12-10T19:28:15,808 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(2659): reportForDuty to master=e4ad70981cbb,41471,1733858893036 with port=41557, startcode=1733858893868 2024-12-10T19:28:15,835 DEBUG [RS:1;e4ad70981cbb:39803 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T19:28:15,835 DEBUG [RS:0;e4ad70981cbb:41557 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T19:28:15,837 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T19:28:15,837 DEBUG [RS:2;e4ad70981cbb:35427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T19:28:15,848 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T19:28:15,856 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T19:28:15,862 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e4ad70981cbb,41471,1733858893036 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T19:28:15,872 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e4ad70981cbb:0, corePoolSize=5, maxPoolSize=5 2024-12-10T19:28:15,872 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e4ad70981cbb:0, corePoolSize=5, maxPoolSize=5 2024-12-10T19:28:15,873 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e4ad70981cbb:0, corePoolSize=5, maxPoolSize=5 2024-12-10T19:28:15,873 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e4ad70981cbb:0, corePoolSize=5, maxPoolSize=5 2024-12-10T19:28:15,873 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e4ad70981cbb:0, corePoolSize=10, maxPoolSize=10 2024-12-10T19:28:15,873 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:15,874 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e4ad70981cbb:0, corePoolSize=2, maxPoolSize=2 2024-12-10T19:28:15,874 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:15,893 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733858925893 2024-12-10T19:28:15,894 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35621, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T19:28:15,895 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35353, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T19:28:15,897 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44703, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T19:28:15,898 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T19:28:15,898 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T19:28:15,898 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T19:28:15,899 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T19:28:15,903 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T19:28:15,903 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T19:28:15,903 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T19:28:15,904 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T19:28:15,905 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:15,906 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:15,908 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T19:28:15,905 INFO [PEWorker-2 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T19:28:15,910 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T19:28:15,912 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T19:28:15,912 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T19:28:15,914 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T19:28:15,914 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T19:28:15,916 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e4ad70981cbb:0:becomeActiveMaster-HFileCleaner.large.0-1733858895916,5,FailOnTimeoutGroup] 2024-12-10T19:28:15,916 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T19:28:15,917 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e4ad70981cbb:0:becomeActiveMaster-HFileCleaner.small.0-1733858895917,5,FailOnTimeoutGroup] 2024-12-10T19:28:15,917 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:15,918 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T19:28:15,918 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T19:28:15,919 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:15,919 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:15,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741831_1007 (size=1321) 2024-12-10T19:28:15,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741831_1007 (size=1321) 2024-12-10T19:28:15,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741831_1007 (size=1321) 2024-12-10T19:28:15,955 INFO [PEWorker-2 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T19:28:15,955 INFO [PEWorker-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36 2024-12-10T19:28:15,964 DEBUG [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-10T19:28:15,964 DEBUG [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-10T19:28:15,964 WARN [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-10T19:28:15,964 WARN [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-10T19:28:15,964 DEBUG [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-10T19:28:15,964 WARN [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-10T19:28:15,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741832_1008 (size=32) 2024-12-10T19:28:15,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741832_1008 (size=32) 2024-12-10T19:28:15,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741832_1008 (size=32) 2024-12-10T19:28:15,974 DEBUG [PEWorker-2 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:15,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T19:28:15,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T19:28:15,980 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:15,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T19:28:15,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T19:28:15,984 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T19:28:15,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:15,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T19:28:15,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T19:28:15,989 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T19:28:15,989 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:15,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T19:28:15,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T19:28:15,994 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T19:28:15,994 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:15,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T19:28:15,995 DEBUG [PEWorker-2 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T19:28:15,997 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740 2024-12-10T19:28:15,998 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740 2024-12-10T19:28:16,002 DEBUG [PEWorker-2 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T19:28:16,002 DEBUG [PEWorker-2 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T19:28:16,003 DEBUG [PEWorker-2 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T19:28:16,006 DEBUG [PEWorker-2 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T19:28:16,010 DEBUG [PEWorker-2 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:16,012 INFO [PEWorker-2 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66202922, jitterRate=-0.013499587774276733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T19:28:16,014 DEBUG [PEWorker-2 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733858895974Initializing all the Stores at 1733858895976 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733858895976Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733858895976Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858895976Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733858895976Cleaning up temporary data from old regions at 1733858896002 (+26 ms)Region opened successfully at 1733858896014 (+12 ms) 2024-12-10T19:28:16,015 DEBUG [PEWorker-2 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T19:28:16,015 INFO [PEWorker-2 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T19:28:16,015 DEBUG [PEWorker-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T19:28:16,015 DEBUG [PEWorker-2 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T19:28:16,015 DEBUG [PEWorker-2 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T19:28:16,017 INFO [PEWorker-2 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T19:28:16,017 DEBUG [PEWorker-2 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733858896015Disabling compacts and flushes for region at 1733858896015Disabling writes for close at 1733858896015Writing region close event to WAL at 1733858896016 (+1 ms)Closed at 1733858896017 (+1 ms) 2024-12-10T19:28:16,021 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T19:28:16,021 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T19:28:16,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T19:28:16,039 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T19:28:16,045 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T19:28:16,066 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(2659): reportForDuty to master=e4ad70981cbb,41471,1733858893036 with port=35427, startcode=1733858894073 2024-12-10T19:28:16,066 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(2659): reportForDuty to master=e4ad70981cbb,41471,1733858893036 with port=41557, startcode=1733858893868 2024-12-10T19:28:16,066 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(2659): reportForDuty to master=e4ad70981cbb,41471,1733858893036 with port=39803, startcode=1733858893985 2024-12-10T19:28:16,068 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:16,072 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] master.ServerManager(517): Registering regionserver=e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:16,085 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:16,086 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] master.ServerManager(517): Registering regionserver=e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:16,086 DEBUG [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36 2024-12-10T19:28:16,087 DEBUG [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46813 2024-12-10T19:28:16,087 DEBUG [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T19:28:16,098 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:16,098 DEBUG [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36 2024-12-10T19:28:16,098 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] master.ServerManager(517): Registering regionserver=e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:16,098 DEBUG [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46813 2024-12-10T19:28:16,098 DEBUG [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T19:28:16,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T19:28:16,105 DEBUG [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36 2024-12-10T19:28:16,105 DEBUG [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46813 2024-12-10T19:28:16,105 DEBUG [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T19:28:16,145 DEBUG [RS:1;e4ad70981cbb:39803 {}] zookeeper.ZKUtil(111): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:16,145 WARN [RS:1;e4ad70981cbb:39803 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T19:28:16,145 INFO [RS:1;e4ad70981cbb:39803 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:16,146 DEBUG [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:16,155 DEBUG [RS:0;e4ad70981cbb:41557 {}] zookeeper.ZKUtil(111): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:16,155 WARN [RS:0;e4ad70981cbb:41557 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T19:28:16,155 INFO [RS:0;e4ad70981cbb:41557 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:16,155 DEBUG [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:16,156 DEBUG [RS:2;e4ad70981cbb:35427 {}] zookeeper.ZKUtil(111): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:16,156 WARN [RS:2;e4ad70981cbb:35427 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T19:28:16,156 INFO [RS:2;e4ad70981cbb:35427 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:16,156 DEBUG [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:16,158 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e4ad70981cbb,39803,1733858893985] 2024-12-10T19:28:16,159 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e4ad70981cbb,35427,1733858894073] 2024-12-10T19:28:16,159 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e4ad70981cbb,41557,1733858893868] 2024-12-10T19:28:16,181 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T19:28:16,181 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T19:28:16,183 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T19:28:16,200 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T19:28:16,204 WARN [e4ad70981cbb:41471 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T19:28:16,204 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T19:28:16,208 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T19:28:16,215 INFO [RS:2;e4ad70981cbb:35427 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T19:28:16,216 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,221 INFO [RS:0;e4ad70981cbb:41557 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T19:28:16,221 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,222 INFO [RS:1;e4ad70981cbb:39803 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T19:28:16,222 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,223 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T19:28:16,228 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T19:28:16,231 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T19:28:16,231 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T19:28:16,233 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,234 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,234 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,235 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,235 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,235 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,235 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0, corePoolSize=2, maxPoolSize=2 2024-12-10T19:28:16,235 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,235 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,236 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,236 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,236 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,236 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,236 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e4ad70981cbb:0, corePoolSize=3, maxPoolSize=3 2024-12-10T19:28:16,236 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,236 DEBUG [RS:0;e4ad70981cbb:41557 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e4ad70981cbb:0, corePoolSize=3, maxPoolSize=3 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0, corePoolSize=2, maxPoolSize=2 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,237 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,238 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,238 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,238 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e4ad70981cbb:0, corePoolSize=3, maxPoolSize=3 2024-12-10T19:28:16,238 DEBUG [RS:2;e4ad70981cbb:35427 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e4ad70981cbb:0, corePoolSize=3, maxPoolSize=3 2024-12-10T19:28:16,244 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T19:28:16,246 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T19:28:16,247 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,247 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,247 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,248 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,248 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,250 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,250 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0, corePoolSize=2, maxPoolSize=2 2024-12-10T19:28:16,250 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,250 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,251 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,251 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,251 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,251 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e4ad70981cbb:0, corePoolSize=1, maxPoolSize=1 2024-12-10T19:28:16,251 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e4ad70981cbb:0, corePoolSize=3, maxPoolSize=3 2024-12-10T19:28:16,251 DEBUG [RS:1;e4ad70981cbb:39803 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e4ad70981cbb:0, corePoolSize=3, maxPoolSize=3 2024-12-10T19:28:16,261 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,261 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,261 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,261 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,262 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,262 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,41557,1733858893868-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T19:28:16,264 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,265 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,265 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,265 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,265 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,265 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,35427,1733858894073-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T19:28:16,293 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T19:28:16,294 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T19:28:16,294 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,294 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,294 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,294 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,295 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,295 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,39803,1733858893985-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T19:28:16,295 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,35427,1733858894073-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,296 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,296 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.Replication(171): e4ad70981cbb,35427,1733858894073 started 2024-12-10T19:28:16,297 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,41557,1733858893868-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,297 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,297 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.Replication(171): e4ad70981cbb,41557,1733858893868 started 2024-12-10T19:28:16,317 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,317 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(1482): Serving as e4ad70981cbb,35427,1733858894073, RpcServer on e4ad70981cbb/172.17.0.3:35427, sessionid=0x100114e4b300003 2024-12-10T19:28:16,318 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T19:28:16,318 DEBUG [RS:2;e4ad70981cbb:35427 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:16,318 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4ad70981cbb,35427,1733858894073' 2024-12-10T19:28:16,319 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T19:28:16,320 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T19:28:16,320 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T19:28:16,320 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T19:28:16,321 DEBUG [RS:2;e4ad70981cbb:35427 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:16,321 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4ad70981cbb,35427,1733858894073' 2024-12-10T19:28:16,321 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T19:28:16,321 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,321 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(1482): Serving as e4ad70981cbb,41557,1733858893868, RpcServer on e4ad70981cbb/172.17.0.3:41557, sessionid=0x100114e4b300001 2024-12-10T19:28:16,321 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T19:28:16,321 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T19:28:16,322 DEBUG [RS:0;e4ad70981cbb:41557 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:16,322 DEBUG [RS:2;e4ad70981cbb:35427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T19:28:16,322 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4ad70981cbb,41557,1733858893868' 2024-12-10T19:28:16,322 INFO [RS:2;e4ad70981cbb:35427 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T19:28:16,322 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T19:28:16,322 INFO [RS:2;e4ad70981cbb:35427 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T19:28:16,323 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T19:28:16,324 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T19:28:16,324 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T19:28:16,324 DEBUG [RS:0;e4ad70981cbb:41557 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:16,324 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4ad70981cbb,41557,1733858893868' 2024-12-10T19:28:16,324 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T19:28:16,325 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T19:28:16,325 DEBUG [RS:0;e4ad70981cbb:41557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T19:28:16,326 INFO [RS:0;e4ad70981cbb:41557 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T19:28:16,326 INFO [RS:0;e4ad70981cbb:41557 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T19:28:16,328 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T19:28:16,328 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,39803,1733858893985-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,328 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,329 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.Replication(171): e4ad70981cbb,39803,1733858893985 started 2024-12-10T19:28:16,347 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:16,348 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(1482): Serving as e4ad70981cbb,39803,1733858893985, RpcServer on e4ad70981cbb/172.17.0.3:39803, sessionid=0x100114e4b300002 2024-12-10T19:28:16,348 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T19:28:16,348 DEBUG [RS:1;e4ad70981cbb:39803 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:16,348 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4ad70981cbb,39803,1733858893985' 2024-12-10T19:28:16,348 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T19:28:16,351 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T19:28:16,352 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T19:28:16,352 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T19:28:16,353 DEBUG [RS:1;e4ad70981cbb:39803 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:16,353 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4ad70981cbb,39803,1733858893985' 2024-12-10T19:28:16,353 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T19:28:16,354 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T19:28:16,355 DEBUG [RS:1;e4ad70981cbb:39803 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T19:28:16,355 INFO [RS:1;e4ad70981cbb:39803 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T19:28:16,355 INFO [RS:1;e4ad70981cbb:39803 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T19:28:16,428 INFO [RS:2;e4ad70981cbb:35427 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:16,428 INFO [RS:0;e4ad70981cbb:41557 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:16,432 INFO [RS:0;e4ad70981cbb:41557 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4ad70981cbb%2C41557%2C1733858893868, suffix=, logDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868, archiveDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/oldWALs, maxLogs=32 2024-12-10T19:28:16,435 INFO [RS:2;e4ad70981cbb:35427 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4ad70981cbb%2C35427%2C1733858894073, suffix=, logDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,35427,1733858894073, archiveDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/oldWALs, maxLogs=32 2024-12-10T19:28:16,450 DEBUG [RS:2;e4ad70981cbb:35427 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,35427,1733858894073/e4ad70981cbb%2C35427%2C1733858894073.1733858896436, exclude list is [], retry=0 2024-12-10T19:28:16,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:16,456 INFO [RS:1;e4ad70981cbb:39803 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:16,456 DEBUG [RS:0;e4ad70981cbb:41557 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868/e4ad70981cbb%2C41557%2C1733858893868.1733858896435, exclude list is [], retry=0 2024-12-10T19:28:16,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:16,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:16,460 INFO [RS:1;e4ad70981cbb:39803 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4ad70981cbb%2C39803%2C1733858893985, suffix=, logDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,39803,1733858893985, archiveDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/oldWALs, maxLogs=32 2024-12-10T19:28:16,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:16,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:16,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:16,486 DEBUG [RS:1;e4ad70981cbb:39803 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,39803,1733858893985/e4ad70981cbb%2C39803%2C1733858893985.1733858896462, exclude list is [], retry=0 2024-12-10T19:28:16,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:16,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:16,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:16,536 INFO [RS:2;e4ad70981cbb:35427 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,35427,1733858894073/e4ad70981cbb%2C35427%2C1733858894073.1733858896436 2024-12-10T19:28:16,544 INFO [RS:0;e4ad70981cbb:41557 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868/e4ad70981cbb%2C41557%2C1733858893868.1733858896435 2024-12-10T19:28:16,561 DEBUG [RS:2;e4ad70981cbb:35427 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539)] 2024-12-10T19:28:16,565 DEBUG [RS:0;e4ad70981cbb:41557 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:28:16,597 INFO [RS:1;e4ad70981cbb:39803 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,39803,1733858893985/e4ad70981cbb%2C39803%2C1733858893985.1733858896462 2024-12-10T19:28:16,598 DEBUG [RS:1;e4ad70981cbb:39803 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:16,707 DEBUG [e4ad70981cbb:41471 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-10T19:28:16,719 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(204): Hosts are {e4ad70981cbb=0} racks are {/default-rack=0} 2024-12-10T19:28:16,730 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T19:28:16,730 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T19:28:16,730 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-10T19:28:16,730 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T19:28:16,731 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T19:28:16,731 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-10T19:28:16,731 INFO [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T19:28:16,731 INFO [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T19:28:16,731 INFO [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-10T19:28:16,731 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T19:28:16,742 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:16,748 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e4ad70981cbb,35427,1733858894073, state=OPENING 2024-12-10T19:28:16,784 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T19:28:16,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:16,792 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:16,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:16,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:28:16,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T19:28:16,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T19:28:16,796 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T19:28:16,796 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T19:28:16,799 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T19:28:16,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e4ad70981cbb,35427,1733858894073}] 2024-12-10T19:28:16,990 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T19:28:16,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37247, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T19:28:17,010 INFO [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T19:28:17,011 INFO [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:17,012 INFO [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T19:28:17,020 INFO [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4ad70981cbb%2C35427%2C1733858894073.meta, suffix=.meta, logDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,35427,1733858894073, archiveDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/oldWALs, maxLogs=32 2024-12-10T19:28:17,040 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,35427,1733858894073/e4ad70981cbb%2C35427%2C1733858894073.meta.1733858897022.meta, exclude list is [], retry=0 2024-12-10T19:28:17,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:17,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:17,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:17,085 INFO [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,35427,1733858894073/e4ad70981cbb%2C35427%2C1733858894073.meta.1733858897022.meta 2024-12-10T19:28:17,088 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539)] 2024-12-10T19:28:17,089 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:17,091 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T19:28:17,095 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T19:28:17,102 INFO [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T19:28:17,108 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T19:28:17,109 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:17,109 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T19:28:17,109 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T19:28:17,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T19:28:17,115 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T19:28:17,115 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:17,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T19:28:17,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T19:28:17,119 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T19:28:17,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:17,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T19:28:17,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T19:28:17,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T19:28:17,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:17,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T19:28:17,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T19:28:17,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T19:28:17,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:17,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T19:28:17,132 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T19:28:17,143 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740 2024-12-10T19:28:17,153 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740 2024-12-10T19:28:17,160 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T19:28:17,160 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T19:28:17,166 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T19:28:17,171 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T19:28:17,176 INFO [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59568950, jitterRate=-0.11235347390174866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T19:28:17,176 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T19:28:17,178 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733858897110Writing region info on filesystem at 1733858897110Initializing all the Stores at 1733858897112 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733858897112Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733858897113 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858897113Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733858897113Cleaning up temporary data from old regions at 1733858897160 (+47 ms)Running coprocessor post-open hooks at 1733858897176 (+16 ms)Region opened successfully at 1733858897178 (+2 ms) 2024-12-10T19:28:17,187 INFO [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733858896978 2024-12-10T19:28:17,209 DEBUG [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T19:28:17,210 INFO [RS_OPEN_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T19:28:17,211 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:17,215 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e4ad70981cbb,35427,1733858894073, state=OPEN 2024-12-10T19:28:17,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T19:28:17,226 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T19:28:17,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T19:28:17,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T19:28:17,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T19:28:17,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T19:28:17,226 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:17,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T19:28:17,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T19:28:17,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T19:28:17,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e4ad70981cbb,35427,1733858894073 in 424 msec 2024-12-10T19:28:17,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T19:28:17,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.2100 sec 2024-12-10T19:28:17,249 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T19:28:17,249 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T19:28:17,270 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T19:28:17,271 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4ad70981cbb,35427,1733858894073, seqNum=-1] 2024-12-10T19:28:17,295 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T19:28:17,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55453, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T19:28:17,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5760 sec 2024-12-10T19:28:17,344 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733858897344, completionTime=-1 2024-12-10T19:28:17,350 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-10T19:28:17,350 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T19:28:17,419 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-10T19:28:17,419 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733858957419 2024-12-10T19:28:17,419 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733859017419 2024-12-10T19:28:17,419 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 69 msec 2024-12-10T19:28:17,421 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T19:28:17,438 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,41471,1733858893036-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:17,439 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,41471,1733858893036-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:17,439 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,41471,1733858893036-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:17,449 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e4ad70981cbb:41471, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:17,450 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:17,455 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:17,468 DEBUG [master/e4ad70981cbb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T19:28:17,494 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.283sec 2024-12-10T19:28:17,496 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T19:28:17,498 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T19:28:17,499 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T19:28:17,500 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T19:28:17,500 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T19:28:17,502 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,41471,1733858893036-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T19:28:17,502 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,41471,1733858893036-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T19:28:17,508 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T19:28:17,510 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T19:28:17,510 INFO [master/e4ad70981cbb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4ad70981cbb,41471,1733858893036-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:17,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed908a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T19:28:17,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4ad70981cbb,41471,-1 for getting cluster id 2024-12-10T19:28:17,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T19:28:17,580 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '896f6e2b-5117-4154-b1cc-9fad4e023b6a' 2024-12-10T19:28:17,586 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T19:28:17,586 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "896f6e2b-5117-4154-b1cc-9fad4e023b6a" 2024-12-10T19:28:17,587 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61cee77f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T19:28:17,587 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4ad70981cbb,41471,-1] 2024-12-10T19:28:17,591 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T19:28:17,594 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:28:17,599 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T19:28:17,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769aece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T19:28:17,606 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T19:28:17,617 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4ad70981cbb,35427,1733858894073, seqNum=-1] 2024-12-10T19:28:17,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T19:28:17,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T19:28:17,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:17,666 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:46813/hbase 2024-12-10T19:28:17,699 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=605, MaxFileDescriptor=1048576, SystemLoadAverage=740, ProcessCount=11, AvailableMemoryMB=3241 2024-12-10T19:28:17,728 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:17,733 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:17,734 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:17,757 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-13749771, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-13749771, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:17,783 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-13749771/hregion-13749771.1733858897759, exclude list is [], retry=0 2024-12-10T19:28:17,794 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:17,801 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:17,812 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:17,850 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-13749771/hregion-13749771.1733858897759 2024-12-10T19:28:17,852 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539)] 2024-12-10T19:28:17,853 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0d4e02c254737336cc90c42c3d639ec2, NAME => 'testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:28:17,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741838_1014 (size=64) 2024-12-10T19:28:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741838_1014 (size=64) 2024-12-10T19:28:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741838_1014 (size=64) 2024-12-10T19:28:17,887 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:17,890 INFO [StoreOpener-0d4e02c254737336cc90c42c3d639ec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:17,893 INFO [StoreOpener-0d4e02c254737336cc90c42c3d639ec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d4e02c254737336cc90c42c3d639ec2 columnFamilyName a 2024-12-10T19:28:17,893 DEBUG [StoreOpener-0d4e02c254737336cc90c42c3d639ec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:17,894 INFO [StoreOpener-0d4e02c254737336cc90c42c3d639ec2-1 {}] regionserver.HStore(327): Store=0d4e02c254737336cc90c42c3d639ec2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:17,896 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:17,903 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:17,905 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:17,906 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:17,907 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:17,910 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:17,914 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:17,915 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d4e02c254737336cc90c42c3d639ec2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60322804, jitterRate=-0.10112017393112183}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T19:28:17,916 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d4e02c254737336cc90c42c3d639ec2: Writing region info on filesystem at 1733858897887Initializing all the Stores at 1733858897889 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858897889Cleaning up temporary data from old regions at 1733858897907 (+18 ms)Region opened successfully at 1733858897916 (+9 ms) 2024-12-10T19:28:17,916 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d4e02c254737336cc90c42c3d639ec2, disabling compactions & flushes 2024-12-10T19:28:17,916 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2. 2024-12-10T19:28:17,916 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2. 2024-12-10T19:28:17,916 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2. after waiting 0 ms 2024-12-10T19:28:17,916 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2. 2024-12-10T19:28:17,917 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2. 2024-12-10T19:28:17,917 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d4e02c254737336cc90c42c3d639ec2: Waiting for close lock at 1733858897916Disabling compacts and flushes for region at 1733858897916Disabling writes for close at 1733858897916Writing region close event to WAL at 1733858897917 (+1 ms)Closed at 1733858897917 2024-12-10T19:28:17,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741837_1013 (size=175) 2024-12-10T19:28:17,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741837_1013 (size=175) 2024-12-10T19:28:17,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741837_1013 (size=175) 2024-12-10T19:28:17,929 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:17,930 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-13749771:(num 1733858897759) 2024-12-10T19:28:17,932 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T19:28:17,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741839_1015 (size=469) 2024-12-10T19:28:17,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741839_1015 (size=469) 2024-12-10T19:28:17,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741839_1015 (size=469) 2024-12-10T19:28:17,984 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T19:28:18,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741840_1016 (size=322) 2024-12-10T19:28:18,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741840_1016 (size=322) 2024-12-10T19:28:18,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741840_1016 (size=322) 2024-12-10T19:28:18,093 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1, size=469 (469bytes) 2024-12-10T19:28:18,094 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-10T19:28:18,094 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-10T19:28:18,094 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1 2024-12-10T19:28:18,103 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1 after 6ms 2024-12-10T19:28:18,176 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:18,177 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1 took 86ms 2024-12-10T19:28:18,184 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1 so closing down 2024-12-10T19:28:18,184 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:18,188 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-10T19:28:18,226 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000001-wal-1.temp 2024-12-10T19:28:18,228 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:18,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741841_1017 (size=469) 2024-12-10T19:28:18,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741841_1017 (size=469) 2024-12-10T19:28:18,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741841_1017 (size=469) 2024-12-10T19:28:18,259 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-10T19:28:18,265 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002 2024-12-10T19:28:18,270 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 86 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-10T19:28:18,271 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1, journal: Splitting hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1, size=469 (469bytes) at 1733858898093Finishing writing output for hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1 so closing down at 1733858898184 (+91 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000001-wal-1.temp at 1733858898226 (+42 ms)3 split writer threads finished at 1733858898228 (+2 ms)Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733858898259 (+31 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002 at 1733858898265 (+6 ms)Processed 2 edits across 1 Regions in 86 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733858898270 (+5 ms) 2024-12-10T19:28:18,295 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2, size=322 (322bytes) 2024-12-10T19:28:18,295 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2 2024-12-10T19:28:18,300 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2 after 5ms 2024-12-10T19:28:18,351 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:18,351 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2 took 57ms 2024-12-10T19:28:18,382 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2 so closing down 2024-12-10T19:28:18,383 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:18,393 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-10T19:28:18,433 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002-wal-2.temp 2024-12-10T19:28:18,434 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:18,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741842_1018 (size=322) 2024-12-10T19:28:18,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741842_1018 (size=322) 2024-12-10T19:28:18,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741842_1018 (size=322) 2024-12-10T19:28:18,469 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-10T19:28:18,517 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:18,521 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-10T19:28:18,530 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 163 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-10T19:28:18,530 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2, journal: Splitting hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2, size=322 (322bytes) at 1733858898295Finishing writing output for hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2 so closing down at 1733858898383 (+88 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002-wal-2.temp at 1733858898433 (+50 ms)3 split writer threads finished at 1733858898434 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733858898469 (+35 ms)Processed 1 edits across 1 Regions in 163 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733858898530 (+61 ms) 2024-12-10T19:28:18,531 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:18,542 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:18,561 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal.1733858898543, exclude list is [], retry=0 2024-12-10T19:28:18,570 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:18,571 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:18,571 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:18,613 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal.1733858898543 2024-12-10T19:28:18,614 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:18,615 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d4e02c254737336cc90c42c3d639ec2, NAME => 'testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:18,615 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:18,615 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:18,615 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:18,620 INFO [StoreOpener-0d4e02c254737336cc90c42c3d639ec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:18,623 INFO [StoreOpener-0d4e02c254737336cc90c42c3d639ec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d4e02c254737336cc90c42c3d639ec2 columnFamilyName a 2024-12-10T19:28:18,623 DEBUG [StoreOpener-0d4e02c254737336cc90c42c3d639ec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:18,624 INFO [StoreOpener-0d4e02c254737336cc90c42c3d639ec2-1 {}] regionserver.HStore(327): Store=0d4e02c254737336cc90c42c3d639ec2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:18,625 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:18,626 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:18,637 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:18,640 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002 2024-12-10T19:28:18,680 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:18,689 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002 2024-12-10T19:28:18,694 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0d4e02c254737336cc90c42c3d639ec2 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-10T19:28:18,786 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/.tmp/a/2c29de49c88c405e820d876c017dcc65 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733858897930/Put/seqid=0 2024-12-10T19:28:18,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741844_1020 (size=5170) 2024-12-10T19:28:18,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741844_1020 (size=5170) 2024-12-10T19:28:18,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741844_1020 (size=5170) 2024-12-10T19:28:18,869 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/.tmp/a/2c29de49c88c405e820d876c017dcc65 2024-12-10T19:28:18,976 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/.tmp/a/2c29de49c88c405e820d876c017dcc65 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/a/2c29de49c88c405e820d876c017dcc65 2024-12-10T19:28:18,996 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/a/2c29de49c88c405e820d876c017dcc65, entries=2, sequenceid=2, filesize=5.0 K 2024-12-10T19:28:19,003 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0d4e02c254737336cc90c42c3d639ec2 in 308ms, sequenceid=2, compaction requested=false; wal=null 2024-12-10T19:28:19,007 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/0000000000000000002 2024-12-10T19:28:19,009 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:19,009 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:19,018 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d4e02c254737336cc90c42c3d639ec2 2024-12-10T19:28:19,028 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/0d4e02c254737336cc90c42c3d639ec2/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-10T19:28:19,044 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d4e02c254737336cc90c42c3d639ec2; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58863004, jitterRate=-0.12287288904190063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T19:28:19,045 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d4e02c254737336cc90c42c3d639ec2: Writing region info on filesystem at 1733858898616Initializing all the Stores at 1733858898618 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858898619 (+1 ms)Obtaining lock to block concurrent updates at 1733858898694 (+75 ms)Preparing flush snapshotting stores in 0d4e02c254737336cc90c42c3d639ec2 at 1733858898694Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733858898698 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733858897729.0d4e02c254737336cc90c42c3d639ec2. at 1733858898699 (+1 ms)Flushing 0d4e02c254737336cc90c42c3d639ec2/a: creating writer at 1733858898701 (+2 ms)Flushing 0d4e02c254737336cc90c42c3d639ec2/a: appending metadata at 1733858898769 (+68 ms)Flushing 0d4e02c254737336cc90c42c3d639ec2/a: closing flushed file at 1733858898773 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9347d94: reopening flushed file at 1733858898961 (+188 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0d4e02c254737336cc90c42c3d639ec2 in 308ms, sequenceid=2, compaction requested=false; wal=null at 1733858899003 (+42 ms)Cleaning up temporary data from old regions at 1733858899009 (+6 ms)Region opened successfully at 1733858899045 (+36 ms) 2024-12-10T19:28:19,092 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 363) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34606 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34572 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:40294 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:41872 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:46813/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:40442 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:41878 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 605) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=740 (was 740), ProcessCount=11 (was 11), AvailableMemoryMB=3308 (was 3241) - AvailableMemoryMB LEAK? - 2024-12-10T19:28:19,115 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=740, ProcessCount=11, AvailableMemoryMB=3307 2024-12-10T19:28:19,138 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:19,142 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:19,144 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:19,150 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73245001, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-73245001, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:19,172 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73245001/hregion-73245001.1733858899151, exclude list is [], retry=0 2024-12-10T19:28:19,189 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:19,201 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:19,201 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:19,289 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73245001/hregion-73245001.1733858899151 2024-12-10T19:28:19,289 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:19,290 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2d25924e72e3b195a8ea81187941813f, NAME => 'testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:28:19,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741846_1022 (size=64) 2024-12-10T19:28:19,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741846_1022 (size=64) 2024-12-10T19:28:19,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741846_1022 (size=64) 2024-12-10T19:28:19,346 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:19,356 INFO [StoreOpener-2d25924e72e3b195a8ea81187941813f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:19,365 INFO [StoreOpener-2d25924e72e3b195a8ea81187941813f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d25924e72e3b195a8ea81187941813f columnFamilyName a 2024-12-10T19:28:19,365 DEBUG [StoreOpener-2d25924e72e3b195a8ea81187941813f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:19,366 INFO [StoreOpener-2d25924e72e3b195a8ea81187941813f-1 {}] regionserver.HStore(327): Store=2d25924e72e3b195a8ea81187941813f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:19,366 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:19,368 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:19,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:19,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:19,370 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:19,377 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:19,386 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:19,387 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d25924e72e3b195a8ea81187941813f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71112505, jitterRate=0.059658899903297424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T19:28:19,388 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d25924e72e3b195a8ea81187941813f: Writing region info on filesystem at 1733858899346Initializing all the Stores at 1733858899350 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858899351 (+1 ms)Cleaning up temporary data from old regions at 1733858899370 (+19 ms)Region opened successfully at 1733858899387 (+17 ms) 2024-12-10T19:28:19,388 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d25924e72e3b195a8ea81187941813f, disabling compactions & flushes 2024-12-10T19:28:19,388 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f. 2024-12-10T19:28:19,388 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f. 2024-12-10T19:28:19,388 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f. after waiting 0 ms 2024-12-10T19:28:19,388 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f. 2024-12-10T19:28:19,389 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f. 2024-12-10T19:28:19,389 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d25924e72e3b195a8ea81187941813f: Waiting for close lock at 1733858899388Disabling compacts and flushes for region at 1733858899388Disabling writes for close at 1733858899388Writing region close event to WAL at 1733858899389 (+1 ms)Closed at 1733858899389 2024-12-10T19:28:19,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741845_1021 (size=175) 2024-12-10T19:28:19,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741845_1021 (size=175) 2024-12-10T19:28:19,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741845_1021 (size=175) 2024-12-10T19:28:19,405 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:19,405 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73245001:(num 1733858899151) 2024-12-10T19:28:19,406 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T19:28:19,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741847_1023 (size=469) 2024-12-10T19:28:19,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741847_1023 (size=469) 2024-12-10T19:28:19,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741847_1023 (size=469) 2024-12-10T19:28:19,509 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T19:28:19,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741848_1024 (size=322) 2024-12-10T19:28:19,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741848_1024 (size=322) 2024-12-10T19:28:19,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741848_1024 (size=322) 2024-12-10T19:28:19,990 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2, size=322 (322bytes) 2024-12-10T19:28:19,990 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2 2024-12-10T19:28:19,991 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2 after 1ms 2024-12-10T19:28:20,020 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:20,020 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2 took 30ms 2024-12-10T19:28:20,023 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2 so closing down 2024-12-10T19:28:20,023 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:20,026 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-10T19:28:20,054 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002-wal-2.temp 2024-12-10T19:28:20,054 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:20,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741849_1025 (size=322) 2024-12-10T19:28:20,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741849_1025 (size=322) 2024-12-10T19:28:20,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741849_1025 (size=322) 2024-12-10T19:28:20,065 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-10T19:28:20,067 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002 2024-12-10T19:28:20,079 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 58 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-10T19:28:20,079 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2, journal: Splitting hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2, size=322 (322bytes) at 1733858899990Finishing writing output for hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2 so closing down at 1733858900023 (+33 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002-wal-2.temp at 1733858900054 (+31 ms)3 split writer threads finished at 1733858900054Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733858900065 (+11 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002 at 1733858900067 (+2 ms)Processed 1 edits across 1 Regions in 58 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733858900079 (+12 ms) 2024-12-10T19:28:20,100 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1, size=469 (469bytes) 2024-12-10T19:28:20,100 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1 2024-12-10T19:28:20,101 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1 after 1ms 2024-12-10T19:28:20,130 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:20,131 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1 took 30ms 2024-12-10T19:28:20,135 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1 so closing down 2024-12-10T19:28:20,135 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:20,138 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-10T19:28:20,163 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000001-wal-1.temp 2024-12-10T19:28:20,163 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:20,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741850_1026 (size=469) 2024-12-10T19:28:20,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741850_1026 (size=469) 2024-12-10T19:28:20,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741850_1026 (size=469) 2024-12-10T19:28:20,185 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-10T19:28:20,217 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:20,220 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002, length=322 2024-12-10T19:28:20,223 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002 2024-12-10T19:28:20,223 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 91 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-10T19:28:20,223 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1, journal: Splitting hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1, size=469 (469bytes) at 1733858900100Finishing writing output for hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1 so closing down at 1733858900135 (+35 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000001-wal-1.temp at 1733858900163 (+28 ms)3 split writer threads finished at 1733858900163Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733858900185 (+22 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002 at 1733858900223 (+38 ms)Processed 2 edits across 1 Regions in 91 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733858900223 2024-12-10T19:28:20,224 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:20,226 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:20,239 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal.1733858900227, exclude list is [], retry=0 2024-12-10T19:28:20,243 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:20,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:20,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:20,277 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal.1733858900227 2024-12-10T19:28:20,280 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:20,280 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d25924e72e3b195a8ea81187941813f, NAME => 'testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:20,281 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:20,281 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:20,281 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:20,292 INFO [StoreOpener-2d25924e72e3b195a8ea81187941813f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:20,296 INFO [StoreOpener-2d25924e72e3b195a8ea81187941813f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d25924e72e3b195a8ea81187941813f columnFamilyName a 2024-12-10T19:28:20,297 DEBUG [StoreOpener-2d25924e72e3b195a8ea81187941813f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:20,298 INFO [StoreOpener-2d25924e72e3b195a8ea81187941813f-1 {}] regionserver.HStore(327): Store=2d25924e72e3b195a8ea81187941813f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:20,299 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:20,300 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:20,304 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:20,306 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002 2024-12-10T19:28:20,346 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:20,349 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002 2024-12-10T19:28:20,349 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d25924e72e3b195a8ea81187941813f 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-10T19:28:20,376 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/.tmp/a/6d1cd8260cb842189907686c561c25ab is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733858899405/Put/seqid=0 2024-12-10T19:28:20,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741852_1028 (size=5170) 2024-12-10T19:28:20,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741852_1028 (size=5170) 2024-12-10T19:28:20,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741852_1028 (size=5170) 2024-12-10T19:28:20,436 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/.tmp/a/6d1cd8260cb842189907686c561c25ab 2024-12-10T19:28:20,449 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/.tmp/a/6d1cd8260cb842189907686c561c25ab as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/a/6d1cd8260cb842189907686c561c25ab 2024-12-10T19:28:20,464 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/a/6d1cd8260cb842189907686c561c25ab, entries=2, sequenceid=2, filesize=5.0 K 2024-12-10T19:28:20,464 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 2d25924e72e3b195a8ea81187941813f in 115ms, sequenceid=2, compaction requested=false; wal=null 2024-12-10T19:28:20,466 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/0000000000000000002 2024-12-10T19:28:20,466 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:20,467 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:20,470 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d25924e72e3b195a8ea81187941813f 2024-12-10T19:28:20,476 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/2d25924e72e3b195a8ea81187941813f/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-10T19:28:20,478 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d25924e72e3b195a8ea81187941813f; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63642878, jitterRate=-0.05164721608161926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T19:28:20,478 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d25924e72e3b195a8ea81187941813f: Writing region info on filesystem at 1733858900281Initializing all the Stores at 1733858900286 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858900286Obtaining lock to block concurrent updates at 1733858900349 (+63 ms)Preparing flush snapshotting stores in 2d25924e72e3b195a8ea81187941813f at 1733858900349Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733858900349Flushing stores of testReplayEditsWrittenIntoWAL,,1733858899139.2d25924e72e3b195a8ea81187941813f. at 1733858900349Flushing 2d25924e72e3b195a8ea81187941813f/a: creating writer at 1733858900349Flushing 2d25924e72e3b195a8ea81187941813f/a: appending metadata at 1733858900374 (+25 ms)Flushing 2d25924e72e3b195a8ea81187941813f/a: closing flushed file at 1733858900374Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d008e6f: reopening flushed file at 1733858900448 (+74 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 2d25924e72e3b195a8ea81187941813f in 115ms, sequenceid=2, compaction requested=false; wal=null at 1733858900464 (+16 ms)Cleaning up temporary data from old regions at 1733858900467 (+3 ms)Region opened successfully at 1733858900478 (+11 ms) 2024-12-10T19:28:20,502 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34572 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:42182 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:40294 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34064 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34180 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=740 (was 740), ProcessCount=11 (was 11), AvailableMemoryMB=3218 (was 3307) 2024-12-10T19:28:20,519 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=740, ProcessCount=11, AvailableMemoryMB=3215 2024-12-10T19:28:20,535 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:20,537 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:20,538 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:20,542 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-04079218, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-04079218, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:20,556 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-04079218/hregion-04079218.1733858900543, exclude list is [], retry=0 2024-12-10T19:28:20,561 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:20,561 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:20,569 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:20,608 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-04079218/hregion-04079218.1733858900543 2024-12-10T19:28:20,609 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539)] 2024-12-10T19:28:20,609 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6a2545ca90ab50037b8f33aa3691e931, NAME => 'testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:28:20,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741854_1030 (size=64) 2024-12-10T19:28:20,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741854_1030 (size=64) 2024-12-10T19:28:20,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741854_1030 (size=64) 2024-12-10T19:28:20,638 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:20,641 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:20,643 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2545ca90ab50037b8f33aa3691e931 columnFamilyName a 2024-12-10T19:28:20,643 DEBUG [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:20,645 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(327): Store=6a2545ca90ab50037b8f33aa3691e931/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:20,645 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:20,648 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2545ca90ab50037b8f33aa3691e931 columnFamilyName b 2024-12-10T19:28:20,648 DEBUG [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:20,650 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(327): Store=6a2545ca90ab50037b8f33aa3691e931/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:20,650 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:20,652 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2545ca90ab50037b8f33aa3691e931 columnFamilyName c 2024-12-10T19:28:20,653 DEBUG [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:20,653 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(327): Store=6a2545ca90ab50037b8f33aa3691e931/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:20,654 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:20,655 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:20,656 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:20,658 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:20,658 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:20,659 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:20,661 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:20,665 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:20,666 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a2545ca90ab50037b8f33aa3691e931; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72651643, jitterRate=0.08259384334087372}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:20,667 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a2545ca90ab50037b8f33aa3691e931: Writing region info on filesystem at 1733858900639Initializing all the Stores at 1733858900640 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858900640Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858900640Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858900640Cleaning up temporary data from old regions at 1733858900658 (+18 ms)Region opened successfully at 1733858900666 (+8 ms) 2024-12-10T19:28:20,667 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a2545ca90ab50037b8f33aa3691e931, disabling compactions & flushes 2024-12-10T19:28:20,667 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. 2024-12-10T19:28:20,667 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. 2024-12-10T19:28:20,667 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. after waiting 0 ms 2024-12-10T19:28:20,667 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. 2024-12-10T19:28:20,667 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. 2024-12-10T19:28:20,667 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a2545ca90ab50037b8f33aa3691e931: Waiting for close lock at 1733858900667Disabling compacts and flushes for region at 1733858900667Disabling writes for close at 1733858900667Writing region close event to WAL at 1733858900667Closed at 1733858900667 2024-12-10T19:28:20,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741853_1029 (size=175) 2024-12-10T19:28:20,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741853_1029 (size=175) 2024-12-10T19:28:20,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741853_1029 (size=175) 2024-12-10T19:28:20,675 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:20,675 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-04079218:(num 1733858900543) 2024-12-10T19:28:20,675 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:20,686 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:20,703 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686, exclude list is [], retry=0 2024-12-10T19:28:20,709 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:20,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:20,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:20,747 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 2024-12-10T19:28:20,747 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:28:21,031 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686, size=0 (0bytes) 2024-12-10T19:28:21,031 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 might be still open, length is 0 2024-12-10T19:28:21,032 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 2024-12-10T19:28:21,034 WARN [IPC Server handler 3 on default port 46813 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-10T19:28:21,034 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 after 2ms 2024-12-10T19:28:21,456 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34082 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39621:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34082 dst: /127.0.0.1:39621 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39621 remote=/127.0.0.1:34082]. Total timeout mills is 60000, 59525 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:21,458 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:42206 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43797:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42206 dst: /127.0.0.1:43797 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:21,461 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34226 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35009:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34226 dst: /127.0.0.1:35009 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:21,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741855_1032 (size=497775) 2024-12-10T19:28:21,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741855_1032 (size=497775) 2024-12-10T19:28:21,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741855_1032 (size=497775) 2024-12-10T19:28:22,620 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T19:28:22,678 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T19:28:23,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T19:28:23,447 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T19:28:23,449 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T19:28:23,450 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T19:28:23,450 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T19:28:23,450 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T19:28:23,452 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-10T19:28:23,452 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-10T19:28:25,036 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 after 4004ms 2024-12-10T19:28:25,083 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:25,086 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 took 4055ms 2024-12-10T19:28:25,109 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733858900686.temp 2024-12-10T19:28:25,170 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000000001-wal.1733858900686.temp 2024-12-10T19:28:25,247 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686; continuing. 2024-12-10T19:28:25,247 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 so closing down 2024-12-10T19:28:25,247 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:25,252 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741856_1033 (size=497783) 2024-12-10T19:28:25,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741856_1033 (size=497783) 2024-12-10T19:28:25,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741856_1033 (size=497783) 2024-12-10T19:28:25,289 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000000001-wal.1733858900686.temp (wrote 3002 edits, skipped 0 edits in 72 ms) 2024-12-10T19:28:25,298 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000000001-wal.1733858900686.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000003002 2024-12-10T19:28:25,299 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 210 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T19:28:25,299 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686, journal: Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686, size=0 (0bytes) at 1733858901031Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000000001-wal.1733858900686.temp at 1733858905170 (+4139 ms)Split 1024 edits, skipped 0 edits. at 1733858905183 (+13 ms)Split 2048 edits, skipped 0 edits. at 1733858905211 (+28 ms)Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 so closing down at 1733858905247 (+36 ms)3 split writer threads finished at 1733858905252 (+5 ms)Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000000001-wal.1733858900686.temp (wrote 3002 edits, skipped 0 edits in 72 ms) at 1733858905289 (+37 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000000001-wal.1733858900686.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000003002 at 1733858905298 (+9 ms)Processed 3002 edits across 1 Regions in 210 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686, size=0, length=0, corrupted=false, cancelled=false at 1733858905299 (+1 ms) 2024-12-10T19:28:25,303 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858900686 2024-12-10T19:28:25,305 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000003002 2024-12-10T19:28:25,305 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:25,310 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:25,331 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858905311, exclude list is [], retry=0 2024-12-10T19:28:25,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:25,351 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:25,351 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:25,396 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858905311 2024-12-10T19:28:25,397 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:25,398 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:25,401 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:25,406 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2545ca90ab50037b8f33aa3691e931 columnFamilyName a 2024-12-10T19:28:25,406 DEBUG [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:25,408 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(327): Store=6a2545ca90ab50037b8f33aa3691e931/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:25,408 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:25,410 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2545ca90ab50037b8f33aa3691e931 columnFamilyName b 2024-12-10T19:28:25,410 DEBUG [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:25,411 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(327): Store=6a2545ca90ab50037b8f33aa3691e931/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:25,411 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:25,416 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2545ca90ab50037b8f33aa3691e931 columnFamilyName c 2024-12-10T19:28:25,416 DEBUG [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:25,417 INFO [StoreOpener-6a2545ca90ab50037b8f33aa3691e931-1 {}] regionserver.HStore(327): Store=6a2545ca90ab50037b8f33aa3691e931/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:25,419 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:25,422 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:25,429 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:25,434 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000003002 2024-12-10T19:28:25,481 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:25,545 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T19:28:26,064 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a2545ca90ab50037b8f33aa3691e931 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-10T19:28:26,115 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/a/711547316dcb429f9162693bf195524f is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733858900757/Put/seqid=0 2024-12-10T19:28:26,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741858_1035 (size=50463) 2024-12-10T19:28:26,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741858_1035 (size=50463) 2024-12-10T19:28:26,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741858_1035 (size=50463) 2024-12-10T19:28:26,141 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/a/711547316dcb429f9162693bf195524f 2024-12-10T19:28:26,163 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/a/711547316dcb429f9162693bf195524f as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/a/711547316dcb429f9162693bf195524f 2024-12-10T19:28:26,175 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/a/711547316dcb429f9162693bf195524f, entries=754, sequenceid=754, filesize=49.3 K 2024-12-10T19:28:26,175 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 6a2545ca90ab50037b8f33aa3691e931 in 111ms, sequenceid=754, compaction requested=false; wal=null 2024-12-10T19:28:26,201 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T19:28:26,201 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a2545ca90ab50037b8f33aa3691e931 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-10T19:28:26,226 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/a/2b8d4be46d864368ac7c73b513c53d65 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733858900804/Put/seqid=0 2024-12-10T19:28:26,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741859_1036 (size=20072) 2024-12-10T19:28:26,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741859_1036 (size=20072) 2024-12-10T19:28:26,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741859_1036 (size=20072) 2024-12-10T19:28:26,259 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/a/2b8d4be46d864368ac7c73b513c53d65 2024-12-10T19:28:26,316 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/b/320cfb3ffbfa446c893f184d3f26a61f is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733858900838/Put/seqid=0 2024-12-10T19:28:26,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741860_1037 (size=35835) 2024-12-10T19:28:26,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741860_1037 (size=35835) 2024-12-10T19:28:26,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741860_1037 (size=35835) 2024-12-10T19:28:26,341 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/b/320cfb3ffbfa446c893f184d3f26a61f 2024-12-10T19:28:26,350 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/a/2b8d4be46d864368ac7c73b513c53d65 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/a/2b8d4be46d864368ac7c73b513c53d65 2024-12-10T19:28:26,361 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/a/2b8d4be46d864368ac7c73b513c53d65, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-10T19:28:26,363 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/b/320cfb3ffbfa446c893f184d3f26a61f as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/b/320cfb3ffbfa446c893f184d3f26a61f 2024-12-10T19:28:26,376 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/b/320cfb3ffbfa446c893f184d3f26a61f, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-10T19:28:26,376 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 6a2545ca90ab50037b8f33aa3691e931 in 175ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-10T19:28:26,394 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T19:28:26,394 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a2545ca90ab50037b8f33aa3691e931 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-10T19:28:26,417 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/b/b7a276b108c14102a89257a7ae0e0677 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733858900870/Put/seqid=0 2024-12-10T19:28:26,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741861_1038 (size=35082) 2024-12-10T19:28:26,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741861_1038 (size=35082) 2024-12-10T19:28:26,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741861_1038 (size=35082) 2024-12-10T19:28:26,505 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/b/b7a276b108c14102a89257a7ae0e0677 2024-12-10T19:28:26,572 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/c/8523d918b7734825a2ef277a4d452b52 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733858900910/Put/seqid=0 2024-12-10T19:28:26,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741862_1039 (size=20825) 2024-12-10T19:28:26,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741862_1039 (size=20825) 2024-12-10T19:28:26,603 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/c/8523d918b7734825a2ef277a4d452b52 2024-12-10T19:28:26,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741862_1039 (size=20825) 2024-12-10T19:28:26,629 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/b/b7a276b108c14102a89257a7ae0e0677 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/b/b7a276b108c14102a89257a7ae0e0677 2024-12-10T19:28:26,650 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/b/b7a276b108c14102a89257a7ae0e0677, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-10T19:28:26,657 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/c/8523d918b7734825a2ef277a4d452b52 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/c/8523d918b7734825a2ef277a4d452b52 2024-12-10T19:28:26,694 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/c/8523d918b7734825a2ef277a4d452b52, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-10T19:28:26,694 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 6a2545ca90ab50037b8f33aa3691e931 in 300ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-10T19:28:26,739 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733858900979/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. 2024-12-10T19:28:26,750 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000003002 2024-12-10T19:28:26,750 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T19:28:26,751 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a2545ca90ab50037b8f33aa3691e931 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-10T19:28:26,818 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/c/ee666bae75af41f5b3b0fd556e16a454 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733858900920/Put/seqid=0 2024-12-10T19:28:26,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741863_1040 (size=50301) 2024-12-10T19:28:26,899 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/c/ee666bae75af41f5b3b0fd556e16a454 2024-12-10T19:28:26,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741863_1040 (size=50301) 2024-12-10T19:28:26,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741863_1040 (size=50301) 2024-12-10T19:28:26,930 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ee666bae75af41f5b3b0fd556e16a454 2024-12-10T19:28:26,932 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/.tmp/c/ee666bae75af41f5b3b0fd556e16a454 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/c/ee666bae75af41f5b3b0fd556e16a454 2024-12-10T19:28:26,982 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ee666bae75af41f5b3b0fd556e16a454 2024-12-10T19:28:26,983 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/c/ee666bae75af41f5b3b0fd556e16a454, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-10T19:28:26,983 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 6a2545ca90ab50037b8f33aa3691e931 in 233ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-10T19:28:26,989 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/0000000000000003002 2024-12-10T19:28:27,009 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:27,009 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:27,017 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T19:28:27,032 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a2545ca90ab50037b8f33aa3691e931 2024-12-10T19:28:27,084 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenIntoWAL/6a2545ca90ab50037b8f33aa3691e931/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-10T19:28:27,096 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a2545ca90ab50037b8f33aa3691e931; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66019666, jitterRate=-0.01623031497001648}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T19:28:27,097 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a2545ca90ab50037b8f33aa3691e931: Writing region info on filesystem at 1733858905398Initializing all the Stores at 1733858905400 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858905400Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858905400Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858905400Cleaning up temporary data from old regions at 1733858907009 (+1609 ms)Region opened successfully at 1733858907097 (+88 ms) 2024-12-10T19:28:27,335 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a2545ca90ab50037b8f33aa3691e931, disabling compactions & flushes 2024-12-10T19:28:27,335 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. 2024-12-10T19:28:27,335 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. 2024-12-10T19:28:27,335 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. after waiting 0 ms 2024-12-10T19:28:27,335 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. 2024-12-10T19:28:27,401 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733858900536.6a2545ca90ab50037b8f33aa3691e931. 2024-12-10T19:28:27,401 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a2545ca90ab50037b8f33aa3691e931: Waiting for close lock at 1733858907335Disabling compacts and flushes for region at 1733858907335Disabling writes for close at 1733858907335Writing region close event to WAL at 1733858907400 (+65 ms)Closed at 1733858907400 2024-12-10T19:28:27,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741857_1034 (size=175) 2024-12-10T19:28:27,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741857_1034 (size=175) 2024-12-10T19:28:27,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741857_1034 (size=175) 2024-12-10T19:28:27,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:27,481 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733858905311) 2024-12-10T19:28:27,507 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=397 (was 386) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-697287048_22 at /127.0.0.1:42274 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:46617 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:46813 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-697287048_22 at /127.0.0.1:34128 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-697287048_22 at /127.0.0.1:34286 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43625 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43625 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-697287048_22 at /127.0.0.1:42242 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:46813 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46617 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=859 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=743 (was 740) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2644 (was 3215) 2024-12-10T19:28:27,530 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=397, OpenFileDescriptor=859, MaxFileDescriptor=1048576, SystemLoadAverage=743, ProcessCount=11, AvailableMemoryMB=2641 2024-12-10T19:28:27,551 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:27,558 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:27,560 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:27,585 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-01240042, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-01240042, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:27,606 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-01240042/hregion-01240042.1733858907586, exclude list is [], retry=0 2024-12-10T19:28:27,625 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:27,625 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:27,626 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:27,709 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-01240042/hregion-01240042.1733858907586 2024-12-10T19:28:27,712 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:28:27,713 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 43bf159dc002a611ed96d6957bd4eb30, NAME => 'test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:28:27,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741865_1042 (size=43) 2024-12-10T19:28:27,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741865_1042 (size=43) 2024-12-10T19:28:27,803 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:27,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741865_1042 (size=43) 2024-12-10T19:28:27,824 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:27,827 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43bf159dc002a611ed96d6957bd4eb30 columnFamilyName a 2024-12-10T19:28:27,827 DEBUG [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:27,828 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(327): Store=43bf159dc002a611ed96d6957bd4eb30/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:27,828 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:27,831 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43bf159dc002a611ed96d6957bd4eb30 columnFamilyName b 2024-12-10T19:28:27,831 DEBUG [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:27,832 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(327): Store=43bf159dc002a611ed96d6957bd4eb30/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:27,832 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:27,834 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43bf159dc002a611ed96d6957bd4eb30 columnFamilyName c 2024-12-10T19:28:27,835 DEBUG [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:27,835 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(327): Store=43bf159dc002a611ed96d6957bd4eb30/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:27,835 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:27,837 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:27,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:27,840 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:27,840 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:27,840 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:27,842 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:27,853 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:27,854 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43bf159dc002a611ed96d6957bd4eb30; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70481691, jitterRate=0.05025903880596161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:27,855 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43bf159dc002a611ed96d6957bd4eb30: Writing region info on filesystem at 1733858907803Initializing all the Stores at 1733858907819 (+16 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858907819Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858907820 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858907820Cleaning up temporary data from old regions at 1733858907840 (+20 ms)Region opened successfully at 1733858907855 (+15 ms) 2024-12-10T19:28:27,855 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 43bf159dc002a611ed96d6957bd4eb30, disabling compactions & flushes 2024-12-10T19:28:27,855 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. 2024-12-10T19:28:27,855 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. 2024-12-10T19:28:27,855 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. after waiting 0 ms 2024-12-10T19:28:27,856 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. 2024-12-10T19:28:27,856 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. 2024-12-10T19:28:27,856 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 43bf159dc002a611ed96d6957bd4eb30: Waiting for close lock at 1733858907855Disabling compacts and flushes for region at 1733858907855Disabling writes for close at 1733858907855Writing region close event to WAL at 1733858907856 (+1 ms)Closed at 1733858907856 2024-12-10T19:28:27,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741864_1041 (size=175) 2024-12-10T19:28:27,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741864_1041 (size=175) 2024-12-10T19:28:27,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741864_1041 (size=175) 2024-12-10T19:28:27,882 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:27,883 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-01240042:(num 1733858907586) 2024-12-10T19:28:27,883 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:27,888 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:27,909 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889, exclude list is [], retry=0 2024-12-10T19:28:27,917 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:27,917 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:27,921 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:27,977 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889 2024-12-10T19:28:27,980 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:28,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741866_1043 (size=371388) 2024-12-10T19:28:28,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741866_1043 (size=371388) 2024-12-10T19:28:28,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741866_1043 (size=371388) 2024-12-10T19:28:28,284 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889, size=362.7 K (371388bytes) 2024-12-10T19:28:28,284 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889 2024-12-10T19:28:28,290 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889 after 6ms 2024-12-10T19:28:28,339 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:28,346 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889 took 63ms 2024-12-10T19:28:28,396 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733858907889.temp 2024-12-10T19:28:28,420 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889 so closing down 2024-12-10T19:28:28,420 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:28,477 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000000001-wal.1733858907889.temp 2024-12-10T19:28:28,522 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:28,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741867_1044 (size=371388) 2024-12-10T19:28:28,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741867_1044 (size=371388) 2024-12-10T19:28:28,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741867_1044 (size=371388) 2024-12-10T19:28:29,019 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000000001-wal.1733858907889.temp (wrote 3000 edits, skipped 0 edits in 44 ms) 2024-12-10T19:28:29,021 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000000001-wal.1733858907889.temp to hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003000 2024-12-10T19:28:29,021 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 665 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-10T19:28:29,021 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889, journal: Splitting hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889, size=362.7 K (371388bytes) at 1733858908284Split 1024 edits, skipped 0 edits. at 1733858908390 (+106 ms)Split 2048 edits, skipped 0 edits. at 1733858908405 (+15 ms)Finishing writing output for hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889 so closing down at 1733858908420 (+15 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000000001-wal.1733858907889.temp at 1733858908478 (+58 ms)3 split writer threads finished at 1733858908522 (+44 ms)Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000000001-wal.1733858907889.temp (wrote 3000 edits, skipped 0 edits in 44 ms) at 1733858909019 (+497 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000000001-wal.1733858907889.temp to hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003000 at 1733858909021 (+2 ms)Processed 3000 edits across 1 Regions in 665 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733858909021 2024-12-10T19:28:29,023 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858907889 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858907889 2024-12-10T19:28:29,024 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003000 2024-12-10T19:28:29,025 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:29,027 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:29,045 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027, exclude list is [], retry=0 2024-12-10T19:28:29,049 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:29,049 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:29,050 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:29,104 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027 2024-12-10T19:28:29,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:28:29,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741868_1045 (size=371515) 2024-12-10T19:28:29,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741868_1045 (size=371515) 2024-12-10T19:28:29,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741868_1045 (size=371515) 2024-12-10T19:28:29,402 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027 not finished, retry = 0 2024-12-10T19:28:29,597 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027, size=362.8 K (371515bytes) 2024-12-10T19:28:29,597 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027 2024-12-10T19:28:29,598 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027 after 1ms 2024-12-10T19:28:29,661 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:29,663 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027 took 67ms 2024-12-10T19:28:29,671 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733858909027.temp 2024-12-10T19:28:29,687 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027 so closing down 2024-12-10T19:28:29,688 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:29,711 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003001-wal.1733858909027.temp 2024-12-10T19:28:29,726 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:29,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741869_1046 (size=371515) 2024-12-10T19:28:29,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741869_1046 (size=371515) 2024-12-10T19:28:29,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741869_1046 (size=371515) 2024-12-10T19:28:29,760 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003001-wal.1733858909027.temp (wrote 3000 edits, skipped 0 edits in 14 ms) 2024-12-10T19:28:29,766 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003001-wal.1733858909027.temp to hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000006000 2024-12-10T19:28:29,766 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-10T19:28:29,766 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027, journal: Splitting hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027, size=362.8 K (371515bytes) at 1733858909597Split 1024 edits, skipped 0 edits. at 1733858909675 (+78 ms)Split 2048 edits, skipped 0 edits. at 1733858909681 (+6 ms)Finishing writing output for hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027 so closing down at 1733858909688 (+7 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003001-wal.1733858909027.temp at 1733858909711 (+23 ms)3 split writer threads finished at 1733858909726 (+15 ms)Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003001-wal.1733858909027.temp (wrote 3000 edits, skipped 0 edits in 14 ms) at 1733858909760 (+34 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003001-wal.1733858909027.temp to hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000006000 at 1733858909766 (+6 ms)Processed 3000 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733858909766 2024-12-10T19:28:29,777 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909027 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858909027 2024-12-10T19:28:29,778 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000006000 2024-12-10T19:28:29,778 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:29,781 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/test2727-manual,16010,1733858907549, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:29,800 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909781, exclude list is [], retry=0 2024-12-10T19:28:29,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:29,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:29,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:29,852 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733858907549/wal.1733858909781 2024-12-10T19:28:29,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:29,857 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 43bf159dc002a611ed96d6957bd4eb30, NAME => 'test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:29,857 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:29,857 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:29,857 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:29,865 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:29,867 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43bf159dc002a611ed96d6957bd4eb30 columnFamilyName a 2024-12-10T19:28:29,867 DEBUG [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:29,869 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(327): Store=43bf159dc002a611ed96d6957bd4eb30/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:29,873 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:29,875 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43bf159dc002a611ed96d6957bd4eb30 columnFamilyName b 2024-12-10T19:28:29,875 DEBUG [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:29,876 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(327): Store=43bf159dc002a611ed96d6957bd4eb30/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:29,877 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:29,886 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43bf159dc002a611ed96d6957bd4eb30 columnFamilyName c 2024-12-10T19:28:29,886 DEBUG [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:29,887 INFO [StoreOpener-43bf159dc002a611ed96d6957bd4eb30-1 {}] regionserver.HStore(327): Store=43bf159dc002a611ed96d6957bd4eb30/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:29,888 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:29,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:29,907 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:29,910 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003000 2024-12-10T19:28:29,950 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:29,977 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003000 2024-12-10T19:28:29,978 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000006000 2024-12-10T19:28:30,004 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:30,033 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000006000 2024-12-10T19:28:30,034 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 43bf159dc002a611ed96d6957bd4eb30 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-10T19:28:30,079 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/.tmp/a/e63cc91c37114479a347ad2ac2defce0 is 41, key is test2727/a:100/1733858909123/Put/seqid=0 2024-12-10T19:28:30,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741871_1048 (size=84227) 2024-12-10T19:28:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741871_1048 (size=84227) 2024-12-10T19:28:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741871_1048 (size=84227) 2024-12-10T19:28:30,099 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/.tmp/a/e63cc91c37114479a347ad2ac2defce0 2024-12-10T19:28:30,155 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/.tmp/b/b69fda89cf644fb19614427aace81a82 is 41, key is test2727/b:100/1733858909248/Put/seqid=0 2024-12-10T19:28:30,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741872_1049 (size=84609) 2024-12-10T19:28:30,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741872_1049 (size=84609) 2024-12-10T19:28:30,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741872_1049 (size=84609) 2024-12-10T19:28:30,176 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/.tmp/b/b69fda89cf644fb19614427aace81a82 2024-12-10T19:28:30,214 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/.tmp/c/a6030530ec4d40e0808ea8bda815e131 is 41, key is test2727/c:100/1733858909332/Put/seqid=0 2024-12-10T19:28:30,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741873_1050 (size=84609) 2024-12-10T19:28:30,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741873_1050 (size=84609) 2024-12-10T19:28:30,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741873_1050 (size=84609) 2024-12-10T19:28:30,260 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/.tmp/c/a6030530ec4d40e0808ea8bda815e131 2024-12-10T19:28:30,284 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/.tmp/a/e63cc91c37114479a347ad2ac2defce0 as hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/a/e63cc91c37114479a347ad2ac2defce0 2024-12-10T19:28:30,301 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/a/e63cc91c37114479a347ad2ac2defce0, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-10T19:28:30,302 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/.tmp/b/b69fda89cf644fb19614427aace81a82 as hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/b/b69fda89cf644fb19614427aace81a82 2024-12-10T19:28:30,312 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/b/b69fda89cf644fb19614427aace81a82, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-10T19:28:30,314 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/.tmp/c/a6030530ec4d40e0808ea8bda815e131 as hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/c/a6030530ec4d40e0808ea8bda815e131 2024-12-10T19:28:30,326 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/c/a6030530ec4d40e0808ea8bda815e131, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-10T19:28:30,327 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 43bf159dc002a611ed96d6957bd4eb30 in 293ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-10T19:28:30,329 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000003000 2024-12-10T19:28:30,330 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/0000000000000006000 2024-12-10T19:28:30,339 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:30,339 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:30,343 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:30,347 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43bf159dc002a611ed96d6957bd4eb30 2024-12-10T19:28:30,356 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/test2727/43bf159dc002a611ed96d6957bd4eb30/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-10T19:28:30,358 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43bf159dc002a611ed96d6957bd4eb30; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71941563, jitterRate=0.07201282680034637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:30,363 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43bf159dc002a611ed96d6957bd4eb30: Writing region info on filesystem at 1733858909857Initializing all the Stores at 1733858909865 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858909865Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858909865Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858909865Obtaining lock to block concurrent updates at 1733858910034 (+169 ms)Preparing flush snapshotting stores in 43bf159dc002a611ed96d6957bd4eb30 at 1733858910034Finished memstore snapshotting test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733858910034Flushing stores of test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. at 1733858910034Flushing 43bf159dc002a611ed96d6957bd4eb30/a: creating writer at 1733858910034Flushing 43bf159dc002a611ed96d6957bd4eb30/a: appending metadata at 1733858910078 (+44 ms)Flushing 43bf159dc002a611ed96d6957bd4eb30/a: closing flushed file at 1733858910078Flushing 43bf159dc002a611ed96d6957bd4eb30/b: creating writer at 1733858910114 (+36 ms)Flushing 43bf159dc002a611ed96d6957bd4eb30/b: appending metadata at 1733858910151 (+37 ms)Flushing 43bf159dc002a611ed96d6957bd4eb30/b: closing flushed file at 1733858910151Flushing 43bf159dc002a611ed96d6957bd4eb30/c: creating writer at 1733858910186 (+35 ms)Flushing 43bf159dc002a611ed96d6957bd4eb30/c: appending metadata at 1733858910213 (+27 ms)Flushing 43bf159dc002a611ed96d6957bd4eb30/c: closing flushed file at 1733858910213Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@530f6a0f: reopening flushed file at 1733858910279 (+66 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@622162b1: reopening flushed file at 1733858910301 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b524759: reopening flushed file at 1733858910312 (+11 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 43bf159dc002a611ed96d6957bd4eb30 in 293ms, sequenceid=6000, compaction requested=false; wal=null at 1733858910327 (+15 ms)Cleaning up temporary data from old regions at 1733858910339 (+12 ms)Region opened successfully at 1733858910363 (+24 ms) 2024-12-10T19:28:30,365 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-10T19:28:30,365 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 43bf159dc002a611ed96d6957bd4eb30, disabling compactions & flushes 2024-12-10T19:28:30,365 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. 2024-12-10T19:28:30,365 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. 2024-12-10T19:28:30,365 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. after waiting 0 ms 2024-12-10T19:28:30,365 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. 2024-12-10T19:28:30,381 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733858907552.43bf159dc002a611ed96d6957bd4eb30. 2024-12-10T19:28:30,382 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 43bf159dc002a611ed96d6957bd4eb30: Waiting for close lock at 1733858910365Disabling compacts and flushes for region at 1733858910365Disabling writes for close at 1733858910365Writing region close event to WAL at 1733858910381 (+16 ms)Closed at 1733858910381 2024-12-10T19:28:30,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741870_1047 (size=175) 2024-12-10T19:28:30,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741870_1047 (size=175) 2024-12-10T19:28:30,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741870_1047 (size=175) 2024-12-10T19:28:30,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:30,423 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733858909781) 2024-12-10T19:28:30,452 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399 (was 397) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34352 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:40098 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:42242 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=921 (was 859) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=743 (was 743), ProcessCount=11 (was 11), AvailableMemoryMB=2476 (was 2641) 2024-12-10T19:28:30,475 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=399, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=743, ProcessCount=11, AvailableMemoryMB=2476 2024-12-10T19:28:30,504 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:30,518 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:30,518 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733858910518 2024-12-10T19:28:30,578 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518 2024-12-10T19:28:30,587 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:28:30,589 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d61a340c89b9d2449ad569493a4fa9e9, NAME => 'testSequentialEditLogSeqNum,,1733858910505.d61a340c89b9d2449ad569493a4fa9e9.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:30,589 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733858910505.d61a340c89b9d2449ad569493a4fa9e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:30,589 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d61a340c89b9d2449ad569493a4fa9e9 2024-12-10T19:28:30,589 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d61a340c89b9d2449ad569493a4fa9e9 2024-12-10T19:28:30,591 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9 doesn't exist for region: d61a340c89b9d2449ad569493a4fa9e9 on table testSequentialEditLogSeqNum 2024-12-10T19:28:30,591 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: d61a340c89b9d2449ad569493a4fa9e9 on table testSequentialEditLogSeqNum 2024-12-10T19:28:30,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741875_1052 (size=62) 2024-12-10T19:28:30,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741875_1052 (size=62) 2024-12-10T19:28:30,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741875_1052 (size=62) 2024-12-10T19:28:30,637 INFO [StoreOpener-d61a340c89b9d2449ad569493a4fa9e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d61a340c89b9d2449ad569493a4fa9e9 2024-12-10T19:28:30,639 INFO [StoreOpener-d61a340c89b9d2449ad569493a4fa9e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d61a340c89b9d2449ad569493a4fa9e9 columnFamilyName a 2024-12-10T19:28:30,640 DEBUG [StoreOpener-d61a340c89b9d2449ad569493a4fa9e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:30,641 INFO [StoreOpener-d61a340c89b9d2449ad569493a4fa9e9-1 {}] regionserver.HStore(327): Store=d61a340c89b9d2449ad569493a4fa9e9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:30,641 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d61a340c89b9d2449ad569493a4fa9e9 2024-12-10T19:28:30,642 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9 2024-12-10T19:28:30,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9 2024-12-10T19:28:30,644 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d61a340c89b9d2449ad569493a4fa9e9 2024-12-10T19:28:30,644 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d61a340c89b9d2449ad569493a4fa9e9 2024-12-10T19:28:30,646 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d61a340c89b9d2449ad569493a4fa9e9 2024-12-10T19:28:30,649 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:30,650 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d61a340c89b9d2449ad569493a4fa9e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73820368, jitterRate=0.10000920295715332}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T19:28:30,651 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d61a340c89b9d2449ad569493a4fa9e9: Writing region info on filesystem at 1733858910590Initializing all the Stores at 1733858910622 (+32 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858910622Cleaning up temporary data from old regions at 1733858910644 (+22 ms)Region opened successfully at 1733858910651 (+7 ms) 2024-12-10T19:28:30,675 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d61a340c89b9d2449ad569493a4fa9e9 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-10T19:28:30,703 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/.tmp/a/5b8f63cc97cd429fa8f106062d9a654e is 81, key is testSequentialEditLogSeqNum/a:x0/1733858910652/Put/seqid=0 2024-12-10T19:28:30,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741876_1053 (size=5833) 2024-12-10T19:28:30,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741876_1053 (size=5833) 2024-12-10T19:28:30,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741876_1053 (size=5833) 2024-12-10T19:28:30,724 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/.tmp/a/5b8f63cc97cd429fa8f106062d9a654e 2024-12-10T19:28:30,741 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/.tmp/a/5b8f63cc97cd429fa8f106062d9a654e as hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/a/5b8f63cc97cd429fa8f106062d9a654e 2024-12-10T19:28:30,749 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/a/5b8f63cc97cd429fa8f106062d9a654e, entries=10, sequenceid=13, filesize=5.7 K 2024-12-10T19:28:30,752 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d61a340c89b9d2449ad569493a4fa9e9 in 76ms, sequenceid=13, compaction requested=false 2024-12-10T19:28:30,752 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d61a340c89b9d2449ad569493a4fa9e9: 2024-12-10T19:28:30,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T19:28:30,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T19:28:30,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T19:28:30,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T19:28:30,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T19:28:30,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741874_1051 (size=3256) 2024-12-10T19:28:30,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741874_1051 (size=3256) 2024-12-10T19:28:30,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741874_1051 (size=3256) 2024-12-10T19:28:31,191 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518, size=3.2 K (3256bytes) 2024-12-10T19:28:31,192 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518 2024-12-10T19:28:31,192 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518 after 0ms 2024-12-10T19:28:31,221 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:31,221 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518 took 30ms 2024-12-10T19:28:31,224 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518 so closing down 2024-12-10T19:28:31,225 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:31,226 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733858910518.temp 2024-12-10T19:28:31,267 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/recovered.edits/0000000000000000003-wal.1733858910518.temp 2024-12-10T19:28:31,271 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:31,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741877_1054 (size=2650) 2024-12-10T19:28:31,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741877_1054 (size=2650) 2024-12-10T19:28:31,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741877_1054 (size=2650) 2024-12-10T19:28:31,298 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/recovered.edits/0000000000000000003-wal.1733858910518.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-10T19:28:31,302 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/recovered.edits/0000000000000000003-wal.1733858910518.temp to hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/recovered.edits/0000000000000000020 2024-12-10T19:28:31,302 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 81 ms; skipped=2; WAL=hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-10T19:28:31,303 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518, journal: Splitting hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518, size=3.2 K (3256bytes) at 1733858911191Finishing writing output for hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518 so closing down at 1733858911224 (+33 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/recovered.edits/0000000000000000003-wal.1733858910518.temp at 1733858911267 (+43 ms)3 split writer threads finished at 1733858911271 (+4 ms)Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/recovered.edits/0000000000000000003-wal.1733858910518.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733858911299 (+28 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/recovered.edits/0000000000000000003-wal.1733858910518.temp to hdfs://localhost:46813/hbase/data/default/testSequentialEditLogSeqNum/d61a340c89b9d2449ad569493a4fa9e9/recovered.edits/0000000000000000020 at 1733858911302 (+3 ms)Processed 17 edits across 1 Regions in 81 ms; skipped=2; WAL=hdfs://localhost:46813/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733858910498/wal.1733858910518, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733858911302 2024-12-10T19:28:31,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=404 (was 399) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34352 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:40098 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=715 (was 743), ProcessCount=11 (was 11), AvailableMemoryMB=2383 (was 2476) 2024-12-10T19:28:31,351 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=404, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=715, ProcessCount=11, AvailableMemoryMB=2378 2024-12-10T19:28:31,372 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:31,375 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:31,413 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:31,416 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39501981, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-39501981, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:31,430 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39501981/hregion-39501981.1733858911416, exclude list is [], retry=0 2024-12-10T19:28:31,436 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:31,438 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:31,440 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:31,477 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39501981/hregion-39501981.1733858911416 2024-12-10T19:28:31,478 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:31,478 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ef50629ac20a19f4a9afb9b476755df0, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:28:31,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741879_1056 (size=70) 2024-12-10T19:28:31,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741879_1056 (size=70) 2024-12-10T19:28:31,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741879_1056 (size=70) 2024-12-10T19:28:31,923 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:31,932 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:31,941 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef50629ac20a19f4a9afb9b476755df0 columnFamilyName a 2024-12-10T19:28:31,941 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:31,946 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(327): Store=ef50629ac20a19f4a9afb9b476755df0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:31,947 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:31,951 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef50629ac20a19f4a9afb9b476755df0 columnFamilyName b 2024-12-10T19:28:31,951 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:31,961 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(327): Store=ef50629ac20a19f4a9afb9b476755df0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:31,961 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:31,970 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef50629ac20a19f4a9afb9b476755df0 columnFamilyName c 2024-12-10T19:28:31,970 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:31,972 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(327): Store=ef50629ac20a19f4a9afb9b476755df0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:31,972 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:31,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:31,979 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:31,985 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:31,985 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:31,993 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:32,000 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,013 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:32,015 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ef50629ac20a19f4a9afb9b476755df0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63662371, jitterRate=-0.05135674774646759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:32,016 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ef50629ac20a19f4a9afb9b476755df0: Writing region info on filesystem at 1733858911923Initializing all the Stores at 1733858911926 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858911926Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858911928 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858911928Cleaning up temporary data from old regions at 1733858911985 (+57 ms)Region opened successfully at 1733858912016 (+31 ms) 2024-12-10T19:28:32,016 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ef50629ac20a19f4a9afb9b476755df0, disabling compactions & flushes 2024-12-10T19:28:32,016 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. 2024-12-10T19:28:32,016 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. 2024-12-10T19:28:32,016 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. after waiting 0 ms 2024-12-10T19:28:32,016 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. 2024-12-10T19:28:32,017 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. 2024-12-10T19:28:32,017 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ef50629ac20a19f4a9afb9b476755df0: Waiting for close lock at 1733858912016Disabling compacts and flushes for region at 1733858912016Disabling writes for close at 1733858912016Writing region close event to WAL at 1733858912017 (+1 ms)Closed at 1733858912017 2024-12-10T19:28:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741878_1055 (size=175) 2024-12-10T19:28:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741878_1055 (size=175) 2024-12-10T19:28:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741878_1055 (size=175) 2024-12-10T19:28:32,041 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-39501981/hregion-39501981.1733858911416 not finished, retry = 0 2024-12-10T19:28:32,146 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:32,146 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39501981:(num 1733858911416) 2024-12-10T19:28:32,146 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:32,150 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:32,171 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150, exclude list is [], retry=0 2024-12-10T19:28:32,179 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:32,180 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:32,180 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:32,229 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 2024-12-10T19:28:32,232 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539)] 2024-12-10T19:28:32,233 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ef50629ac20a19f4a9afb9b476755df0, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:32,233 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:32,233 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,233 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,242 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,247 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef50629ac20a19f4a9afb9b476755df0 columnFamilyName a 2024-12-10T19:28:32,247 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:32,248 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(327): Store=ef50629ac20a19f4a9afb9b476755df0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:32,248 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,253 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef50629ac20a19f4a9afb9b476755df0 columnFamilyName b 2024-12-10T19:28:32,253 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:32,254 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(327): Store=ef50629ac20a19f4a9afb9b476755df0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:32,255 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,256 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef50629ac20a19f4a9afb9b476755df0 columnFamilyName c 2024-12-10T19:28:32,256 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:32,257 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(327): Store=ef50629ac20a19f4a9afb9b476755df0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:32,258 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,259 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,261 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,262 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,262 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,263 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:32,265 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:32,266 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ef50629ac20a19f4a9afb9b476755df0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73723698, jitterRate=0.09856870770454407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:32,267 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ef50629ac20a19f4a9afb9b476755df0: Writing region info on filesystem at 1733858912233Initializing all the Stores at 1733858912239 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858912239Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858912240 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858912240Cleaning up temporary data from old regions at 1733858912262 (+22 ms)Region opened successfully at 1733858912267 (+5 ms) 2024-12-10T19:28:32,272 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733858912270/Put/seqid=0 2024-12-10T19:28:32,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741881_1058 (size=4826) 2024-12-10T19:28:32,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741881_1058 (size=4826) 2024-12-10T19:28:32,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741881_1058 (size=4826) 2024-12-10T19:28:32,685 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46813/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in ef50629ac20a19f4a9afb9b476755df0/a 2024-12-10T19:28:32,694 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-10T19:28:32,695 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T19:28:32,695 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ef50629ac20a19f4a9afb9b476755df0: 2024-12-10T19:28:32,697 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/a/416a7880e79b4da9bc4dd320d81800fb_SeqId_3_ 2024-12-10T19:28:32,698 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46813/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into ef50629ac20a19f4a9afb9b476755df0/a as hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/a/416a7880e79b4da9bc4dd320d81800fb_SeqId_3_ - updating store file list. 2024-12-10T19:28:32,704 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 416a7880e79b4da9bc4dd320d81800fb_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-10T19:28:32,705 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/a/416a7880e79b4da9bc4dd320d81800fb_SeqId_3_ into ef50629ac20a19f4a9afb9b476755df0/a 2024-12-10T19:28:32,705 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46813/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into ef50629ac20a19f4a9afb9b476755df0/a (new location: hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/a/416a7880e79b4da9bc4dd320d81800fb_SeqId_3_) 2024-12-10T19:28:32,755 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150, size=0 (0bytes) 2024-12-10T19:28:32,755 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 might be still open, length is 0 2024-12-10T19:28:32,755 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 2024-12-10T19:28:32,757 WARN [IPC Server handler 2 on default port 46813 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-10T19:28:32,757 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 after 2ms 2024-12-10T19:28:33,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-10T19:28:33,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-10T19:28:33,448 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-10T19:28:33,448 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-10T19:28:33,467 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:41804 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43797:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41804 dst: /127.0.0.1:43797 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43797 remote=/127.0.0.1:41804]. Total timeout mills is 60000, 59246 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:33,470 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:40170 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39621:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40170 dst: /127.0.0.1:39621 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:33,470 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:54576 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35009:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54576 dst: /127.0.0.1:35009 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:33,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741880_1059 (size=606) 2024-12-10T19:28:36,758 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 after 4003ms 2024-12-10T19:28:36,783 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:36,783 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 took 4028ms 2024-12-10T19:28:36,786 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150; continuing. 2024-12-10T19:28:36,786 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 so closing down 2024-12-10T19:28:36,786 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:36,787 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733858912150.temp 2024-12-10T19:28:36,810 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005-wal.1733858912150.temp 2024-12-10T19:28:36,810 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:36,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741882_1060 (size=329) 2024-12-10T19:28:36,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741882_1060 (size=329) 2024-12-10T19:28:36,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741882_1060 (size=329) 2024-12-10T19:28:36,818 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005-wal.1733858912150.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-10T19:28:36,820 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005-wal.1733858912150.temp to hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005 2024-12-10T19:28:36,820 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T19:28:36,820 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150, journal: Splitting hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150, size=0 (0bytes) at 1733858912755Finishing writing output for hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 so closing down at 1733858916786 (+4031 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005-wal.1733858912150.temp at 1733858916810 (+24 ms)3 split writer threads finished at 1733858916810Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005-wal.1733858912150.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733858916818 (+8 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005-wal.1733858912150.temp to hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005 at 1733858916820 (+2 ms)Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150, size=0, length=0, corrupted=false, cancelled=false at 1733858916820 2024-12-10T19:28:36,822 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858912150 2024-12-10T19:28:36,822 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005 2024-12-10T19:28:36,823 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:36,825 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:36,839 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858916825, exclude list is [], retry=0 2024-12-10T19:28:36,842 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:36,842 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:36,842 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:36,869 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858916825 2024-12-10T19:28:36,869 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:28:36,869 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ef50629ac20a19f4a9afb9b476755df0, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:36,870 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:36,870 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,870 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,874 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,875 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef50629ac20a19f4a9afb9b476755df0 columnFamilyName a 2024-12-10T19:28:36,876 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:36,883 DEBUG [StoreFileOpener-ef50629ac20a19f4a9afb9b476755df0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 416a7880e79b4da9bc4dd320d81800fb_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-10T19:28:36,883 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/a/416a7880e79b4da9bc4dd320d81800fb_SeqId_3_ 2024-12-10T19:28:36,883 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(327): Store=ef50629ac20a19f4a9afb9b476755df0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:36,884 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,885 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef50629ac20a19f4a9afb9b476755df0 columnFamilyName b 2024-12-10T19:28:36,885 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:36,886 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(327): Store=ef50629ac20a19f4a9afb9b476755df0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:36,886 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,887 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef50629ac20a19f4a9afb9b476755df0 columnFamilyName c 2024-12-10T19:28:36,887 DEBUG [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:36,888 INFO [StoreOpener-ef50629ac20a19f4a9afb9b476755df0-1 {}] regionserver.HStore(327): Store=ef50629ac20a19f4a9afb9b476755df0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:36,888 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,889 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,890 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,891 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005 2024-12-10T19:28:36,926 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:36,927 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005 2024-12-10T19:28:36,927 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ef50629ac20a19f4a9afb9b476755df0 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-10T19:28:36,941 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/.tmp/a/3abdf12b2cdd451f9349d5e3dba573d4 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733858912711/Put/seqid=0 2024-12-10T19:28:36,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741884_1062 (size=5149) 2024-12-10T19:28:36,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741884_1062 (size=5149) 2024-12-10T19:28:36,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741884_1062 (size=5149) 2024-12-10T19:28:36,950 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/.tmp/a/3abdf12b2cdd451f9349d5e3dba573d4 2024-12-10T19:28:36,957 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/.tmp/a/3abdf12b2cdd451f9349d5e3dba573d4 as hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/a/3abdf12b2cdd451f9349d5e3dba573d4 2024-12-10T19:28:36,963 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/a/3abdf12b2cdd451f9349d5e3dba573d4, entries=1, sequenceid=5, filesize=5.0 K 2024-12-10T19:28:36,963 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for ef50629ac20a19f4a9afb9b476755df0 in 36ms, sequenceid=5, compaction requested=false; wal=null 2024-12-10T19:28:36,964 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/0000000000000000005 2024-12-10T19:28:36,966 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,966 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,967 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:36,968 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ef50629ac20a19f4a9afb9b476755df0 2024-12-10T19:28:36,971 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ef50629ac20a19f4a9afb9b476755df0/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-10T19:28:36,972 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ef50629ac20a19f4a9afb9b476755df0; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74282418, jitterRate=0.10689428448677063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:36,972 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ef50629ac20a19f4a9afb9b476755df0: Writing region info on filesystem at 1733858916870Initializing all the Stores at 1733858916871 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858916871Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858916874 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858916874Obtaining lock to block concurrent updates at 1733858916927 (+53 ms)Preparing flush snapshotting stores in ef50629ac20a19f4a9afb9b476755df0 at 1733858916927Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733858916927Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. at 1733858916927Flushing ef50629ac20a19f4a9afb9b476755df0/a: creating writer at 1733858916927Flushing ef50629ac20a19f4a9afb9b476755df0/a: appending metadata at 1733858916940 (+13 ms)Flushing ef50629ac20a19f4a9afb9b476755df0/a: closing flushed file at 1733858916940Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c5171fb: reopening flushed file at 1733858916956 (+16 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for ef50629ac20a19f4a9afb9b476755df0 in 36ms, sequenceid=5, compaction requested=false; wal=null at 1733858916963 (+7 ms)Cleaning up temporary data from old regions at 1733858916966 (+3 ms)Region opened successfully at 1733858916972 (+6 ms) 2024-12-10T19:28:36,976 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ef50629ac20a19f4a9afb9b476755df0, disabling compactions & flushes 2024-12-10T19:28:36,976 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. 2024-12-10T19:28:36,976 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. 2024-12-10T19:28:36,976 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. after waiting 0 ms 2024-12-10T19:28:36,976 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. 2024-12-10T19:28:36,977 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733858911373.ef50629ac20a19f4a9afb9b476755df0. 2024-12-10T19:28:36,977 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ef50629ac20a19f4a9afb9b476755df0: Waiting for close lock at 1733858916976Disabling compacts and flushes for region at 1733858916976Disabling writes for close at 1733858916976Writing region close event to WAL at 1733858916977 (+1 ms)Closed at 1733858916977 2024-12-10T19:28:36,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741883_1061 (size=175) 2024-12-10T19:28:36,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741883_1061 (size=175) 2024-12-10T19:28:36,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741883_1061 (size=175) 2024-12-10T19:28:36,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:36,982 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733858916825) 2024-12-10T19:28:36,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 404) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:46813 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:46813 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_607330382_22 at /127.0.0.1:40206 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_607330382_22 at /127.0.0.1:41826 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_607330382_22 at /127.0.0.1:54588 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=682 (was 715), ProcessCount=11 (was 11), AvailableMemoryMB=2616 (was 2378) - AvailableMemoryMB LEAK? - 2024-12-10T19:28:37,004 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=682, ProcessCount=11, AvailableMemoryMB=2615 2024-12-10T19:28:37,016 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:37,020 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T19:28:37,025 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:37,027 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a4abb72 2024-12-10T19:28:37,028 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T19:28:37,030 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T19:28:37,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T19:28:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-10T19:28:37,041 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T19:28:37,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-10T19:28:37,043 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:37,045 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T19:28:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T19:28:37,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741885_1063 (size=694) 2024-12-10T19:28:37,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741885_1063 (size=694) 2024-12-10T19:28:37,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741885_1063 (size=694) 2024-12-10T19:28:37,058 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 15dfd37ce6f5438f845b062fa5ab3738, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36 2024-12-10T19:28:37,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741886_1064 (size=77) 2024-12-10T19:28:37,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741886_1064 (size=77) 2024-12-10T19:28:37,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741886_1064 (size=77) 2024-12-10T19:28:37,071 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:37,071 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 15dfd37ce6f5438f845b062fa5ab3738, disabling compactions & flushes 2024-12-10T19:28:37,071 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,071 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,071 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. after waiting 0 ms 2024-12-10T19:28:37,071 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,071 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,071 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 15dfd37ce6f5438f845b062fa5ab3738: Waiting for close lock at 1733858917071Disabling compacts and flushes for region at 1733858917071Disabling writes for close at 1733858917071Writing region close event to WAL at 1733858917071Closed at 1733858917071 2024-12-10T19:28:37,074 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T19:28:37,080 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733858917074"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733858917074"}]},"ts":"1733858917074"} 2024-12-10T19:28:37,086 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T19:28:37,088 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T19:28:37,091 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733858917088"}]},"ts":"1733858917088"} 2024-12-10T19:28:37,096 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-10T19:28:37,097 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {e4ad70981cbb=0} racks are {/default-rack=0} 2024-12-10T19:28:37,098 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T19:28:37,099 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T19:28:37,099 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-10T19:28:37,099 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T19:28:37,099 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T19:28:37,099 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-10T19:28:37,099 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T19:28:37,099 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T19:28:37,099 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-10T19:28:37,099 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T19:28:37,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN}] 2024-12-10T19:28:37,103 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN 2024-12-10T19:28:37,105 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN; state=OFFLINE, location=e4ad70981cbb,41557,1733858893868; forceNewPlan=false, retain=false 2024-12-10T19:28:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T19:28:37,257 INFO [e4ad70981cbb:41471 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T19:28:37,258 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=OPENING, regionLocation=e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:37,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN because future has completed 2024-12-10T19:28:37,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,41557,1733858893868}] 2024-12-10T19:28:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T19:28:37,416 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T19:28:37,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45119, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T19:28:37,432 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,432 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 15dfd37ce6f5438f845b062fa5ab3738, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:37,433 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,433 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:37,434 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,434 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,436 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,439 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf1 2024-12-10T19:28:37,439 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:37,439 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:37,440 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,442 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf2 2024-12-10T19:28:37,442 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:37,445 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:37,446 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,447 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,447 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,449 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,449 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,450 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T19:28:37,453 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,460 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:37,461 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 15dfd37ce6f5438f845b062fa5ab3738; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70876810, jitterRate=0.0561467707157135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T19:28:37,461 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,462 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 15dfd37ce6f5438f845b062fa5ab3738: Running coprocessor pre-open hook at 1733858917434Writing region info on filesystem at 1733858917434Initializing all the Stores at 1733858917436 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858917436Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858917436Cleaning up temporary data from old regions at 1733858917449 (+13 ms)Running coprocessor post-open hooks at 1733858917461 (+12 ms)Region opened successfully at 1733858917462 (+1 ms) 2024-12-10T19:28:37,465 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., pid=6, masterSystemTime=1733858917416 2024-12-10T19:28:37,469 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,469 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,471 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=OPEN, openSeqNum=2, regionLocation=e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:37,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,41557,1733858893868 because future has completed 2024-12-10T19:28:37,475 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41471 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e4ad70981cbb,41557,1733858893868, table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-10T19:28:37,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T19:28:37,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,41557,1733858893868 in 217 msec 2024-12-10T19:28:37,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T19:28:37,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN in 382 msec 2024-12-10T19:28:37,495 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T19:28:37,495 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733858917495"}]},"ts":"1733858917495"} 2024-12-10T19:28:37,500 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-10T19:28:37,503 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T19:28:37,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 468 msec 2024-12-10T19:28:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T19:28:37,690 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-10T19:28:37,693 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-10T19:28:37,694 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T19:28:37,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-10T19:28:37,709 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T19:28:37,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-10T19:28:37,732 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=2] 2024-12-10T19:28:37,734 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T19:28:37,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T19:28:37,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=15dfd37ce6f5438f845b062fa5ab3738, source=e4ad70981cbb,41557,1733858893868, destination=e4ad70981cbb,39803,1733858893985, warming up region on e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T19:28:37,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=15dfd37ce6f5438f845b062fa5ab3738, source=e4ad70981cbb,41557,1733858893868, destination=e4ad70981cbb,39803,1733858893985, running balancer 2024-12-10T19:28:37,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38765, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T19:28:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE 2024-12-10T19:28:37,825 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE 2024-12-10T19:28:37,830 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=CLOSING, regionLocation=e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:37,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.HRegion(7855): Warmup {ENCODED => 15dfd37ce6f5438f845b062fa5ab3738, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:37,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE because future has completed 2024-12-10T19:28:37,839 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,840 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T19:28:37,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,41557,1733858893868}] 2024-12-10T19:28:37,841 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf1 2024-12-10T19:28:37,841 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:37,845 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:37,845 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:37,846 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf2 2024-12-10T19:28:37,846 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:37,848 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.HRegion(1722): Closing 15dfd37ce6f5438f845b062fa5ab3738, disabling compactions & flushes 2024-12-10T19:28:37,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. after waiting 0 ms 2024-12-10T19:28:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] regionserver.HRegion(1676): Region close journal for 15dfd37ce6f5438f845b062fa5ab3738: Waiting for close lock at 1733858917848Disabling compacts and flushes for region at 1733858917848Disabling writes for close at 1733858917848Writing region close event to WAL at 1733858917849 (+1 ms)Closed at 1733858917849 2024-12-10T19:28:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-10T19:28:38,008 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,010 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T19:28:38,011 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 15dfd37ce6f5438f845b062fa5ab3738, disabling compactions & flushes 2024-12-10T19:28:38,011 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:38,011 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:38,011 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. after waiting 0 ms 2024-12-10T19:28:38,011 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:38,011 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 15dfd37ce6f5438f845b062fa5ab3738 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-10T19:28:38,063 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf1/a84360d40fd340ea8969fd7274a679b1 is 35, key is r1/cf1:q/1733858917760/Put/seqid=0 2024-12-10T19:28:38,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741887_1065 (size=4783) 2024-12-10T19:28:38,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741887_1065 (size=4783) 2024-12-10T19:28:38,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741887_1065 (size=4783) 2024-12-10T19:28:38,138 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf1/a84360d40fd340ea8969fd7274a679b1 2024-12-10T19:28:38,174 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf1/a84360d40fd340ea8969fd7274a679b1 as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/a84360d40fd340ea8969fd7274a679b1 2024-12-10T19:28:38,191 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/a84360d40fd340ea8969fd7274a679b1, entries=1, sequenceid=5, filesize=4.7 K 2024-12-10T19:28:38,194 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 15dfd37ce6f5438f845b062fa5ab3738 in 183ms, sequenceid=5, compaction requested=false 2024-12-10T19:28:38,196 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-10T19:28:38,208 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-10T19:28:38,213 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:38,216 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 15dfd37ce6f5438f845b062fa5ab3738: Waiting for close lock at 1733858918011Running coprocessor pre-close hooks at 1733858918011Disabling compacts and flushes for region at 1733858918011Disabling writes for close at 1733858918011Obtaining lock to block concurrent updates at 1733858918011Preparing flush snapshotting stores in 15dfd37ce6f5438f845b062fa5ab3738 at 1733858918011Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733858918012 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. at 1733858918017 (+5 ms)Flushing 15dfd37ce6f5438f845b062fa5ab3738/cf1: creating writer at 1733858918017Flushing 15dfd37ce6f5438f845b062fa5ab3738/cf1: appending metadata at 1733858918062 (+45 ms)Flushing 15dfd37ce6f5438f845b062fa5ab3738/cf1: closing flushed file at 1733858918062Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@756ed86d: reopening flushed file at 1733858918168 (+106 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 15dfd37ce6f5438f845b062fa5ab3738 in 183ms, sequenceid=5, compaction requested=false at 1733858918194 (+26 ms)Writing region close event to WAL at 1733858918201 (+7 ms)Running coprocessor post-close hooks at 1733858918210 (+9 ms)Closed at 1733858918212 (+2 ms) 2024-12-10T19:28:38,217 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 15dfd37ce6f5438f845b062fa5ab3738 move to e4ad70981cbb,39803,1733858893985 record at close sequenceid=5 2024-12-10T19:28:38,221 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,222 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=CLOSED 2024-12-10T19:28:38,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,41557,1733858893868 because future has completed 2024-12-10T19:28:38,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T19:28:38,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,41557,1733858893868 in 390 msec 2024-12-10T19:28:38,240 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE; state=CLOSED, location=e4ad70981cbb,39803,1733858893985; forceNewPlan=false, retain=false 2024-12-10T19:28:38,390 INFO [e4ad70981cbb:41471 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T19:28:38,390 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=OPENING, regionLocation=e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:38,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE because future has completed 2024-12-10T19:28:38,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,39803,1733858893985}] 2024-12-10T19:28:38,556 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:38,556 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 15dfd37ce6f5438f845b062fa5ab3738, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:38,557 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,557 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:38,557 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,557 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,561 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,562 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf1 2024-12-10T19:28:38,562 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:38,571 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/a84360d40fd340ea8969fd7274a679b1 2024-12-10T19:28:38,571 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:38,571 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,574 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf2 2024-12-10T19:28:38,574 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:38,575 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:38,575 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,577 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,579 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,580 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,580 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,586 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T19:28:38,589 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,591 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 15dfd37ce6f5438f845b062fa5ab3738; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68146604, jitterRate=0.015463531017303467}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T19:28:38,592 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:38,593 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 15dfd37ce6f5438f845b062fa5ab3738: Running coprocessor pre-open hook at 1733858918557Writing region info on filesystem at 1733858918557Initializing all the Stores at 1733858918559 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858918559Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858918560 (+1 ms)Cleaning up temporary data from old regions at 1733858918580 (+20 ms)Running coprocessor post-open hooks at 1733858918592 (+12 ms)Region opened successfully at 1733858918593 (+1 ms) 2024-12-10T19:28:38,595 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., pid=9, masterSystemTime=1733858918549 2024-12-10T19:28:38,600 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:38,600 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=OPEN, openSeqNum=9, regionLocation=e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:38,600 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:38,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,39803,1733858893985 because future has completed 2024-12-10T19:28:38,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-10T19:28:38,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,39803,1733858893985 in 214 msec 2024-12-10T19:28:38,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE in 812 msec 2024-12-10T19:28:38,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T19:28:38,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T19:28:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:44858 deadline: 1733858978669, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e4ad70981cbb port=39803 startCode=1733858893985. As of locationSeqNum=5. 2024-12-10T19:28:38,680 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e4ad70981cbb port=39803 startCode=1733858893985. As of locationSeqNum=5. 2024-12-10T19:28:38,681 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e4ad70981cbb port=39803 startCode=1733858893985. As of locationSeqNum=5. 2024-12-10T19:28:38,681 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,39803,1733858893985, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e4ad70981cbb port=39803 startCode=1733858893985. As of locationSeqNum=5. 2024-12-10T19:28:38,788 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T19:28:38,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T19:28:38,808 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 15dfd37ce6f5438f845b062fa5ab3738 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-10T19:28:38,838 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf1/6139fb999dcc426ba693d2891791f485 is 29, key is r1/cf1:/1733858918795/DeleteFamily/seqid=0 2024-12-10T19:28:38,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741888_1066 (size=4906) 2024-12-10T19:28:38,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741888_1066 (size=4906) 2024-12-10T19:28:38,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741888_1066 (size=4906) 2024-12-10T19:28:38,884 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf1/6139fb999dcc426ba693d2891791f485 2024-12-10T19:28:38,909 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6139fb999dcc426ba693d2891791f485 2024-12-10T19:28:38,946 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf2/26387fa911c44cb6bd4b8a6495e7370a is 29, key is r1/cf2:/1733858918795/DeleteFamily/seqid=0 2024-12-10T19:28:38,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741889_1067 (size=4906) 2024-12-10T19:28:38,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741889_1067 (size=4906) 2024-12-10T19:28:39,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741889_1067 (size=4906) 2024-12-10T19:28:39,000 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf2/26387fa911c44cb6bd4b8a6495e7370a 2024-12-10T19:28:39,016 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26387fa911c44cb6bd4b8a6495e7370a 2024-12-10T19:28:39,023 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf1/6139fb999dcc426ba693d2891791f485 as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/6139fb999dcc426ba693d2891791f485 2024-12-10T19:28:39,044 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6139fb999dcc426ba693d2891791f485 2024-12-10T19:28:39,044 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/6139fb999dcc426ba693d2891791f485, entries=1, sequenceid=12, filesize=4.8 K 2024-12-10T19:28:39,047 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf2/26387fa911c44cb6bd4b8a6495e7370a as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/26387fa911c44cb6bd4b8a6495e7370a 2024-12-10T19:28:39,064 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26387fa911c44cb6bd4b8a6495e7370a 2024-12-10T19:28:39,064 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/26387fa911c44cb6bd4b8a6495e7370a, entries=1, sequenceid=12, filesize=4.8 K 2024-12-10T19:28:39,066 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 15dfd37ce6f5438f845b062fa5ab3738 in 258ms, sequenceid=12, compaction requested=false 2024-12-10T19:28:39,066 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 15dfd37ce6f5438f845b062fa5ab3738: 2024-12-10T19:28:39,070 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T19:28:39,072 DEBUG [Time-limited test {}] regionserver.HStore(1541): 15dfd37ce6f5438f845b062fa5ab3738/cf1 is initiating major compaction (all files) 2024-12-10T19:28:39,072 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T19:28:39,072 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:39,073 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 15dfd37ce6f5438f845b062fa5ab3738/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,074 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/a84360d40fd340ea8969fd7274a679b1, hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/6139fb999dcc426ba693d2891791f485] into tmpdir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp, totalSize=9.5 K 2024-12-10T19:28:39,075 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a84360d40fd340ea8969fd7274a679b1, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733858917760 2024-12-10T19:28:39,077 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6139fb999dcc426ba693d2891791f485, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-10T19:28:39,096 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 15dfd37ce6f5438f845b062fa5ab3738#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T19:28:39,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741890_1068 (size=4626) 2024-12-10T19:28:39,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741890_1068 (size=4626) 2024-12-10T19:28:39,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741890_1068 (size=4626) 2024-12-10T19:28:39,137 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf1/82da3c517cfb4855a9db02ea2712a09f as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/82da3c517cfb4855a9db02ea2712a09f 2024-12-10T19:28:39,159 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 15dfd37ce6f5438f845b062fa5ab3738/cf1 of 15dfd37ce6f5438f845b062fa5ab3738 into 82da3c517cfb4855a9db02ea2712a09f(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T19:28:39,159 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 15dfd37ce6f5438f845b062fa5ab3738: 2024-12-10T19:28:39,159 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-10T19:28:39,159 DEBUG [Time-limited test {}] regionserver.HStore(1541): 15dfd37ce6f5438f845b062fa5ab3738/cf2 is initiating major compaction (all files) 2024-12-10T19:28:39,160 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T19:28:39,160 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T19:28:39,160 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 15dfd37ce6f5438f845b062fa5ab3738/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,160 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/26387fa911c44cb6bd4b8a6495e7370a] into tmpdir=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp, totalSize=4.8 K 2024-12-10T19:28:39,162 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 26387fa911c44cb6bd4b8a6495e7370a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-10T19:28:39,172 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 15dfd37ce6f5438f845b062fa5ab3738#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T19:28:39,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741891_1069 (size=4592) 2024-12-10T19:28:39,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741891_1069 (size=4592) 2024-12-10T19:28:39,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741891_1069 (size=4592) 2024-12-10T19:28:39,201 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/.tmp/cf2/ff9e415bfb4e4c49aa98b5a5ca8819d6 as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/ff9e415bfb4e4c49aa98b5a5ca8819d6 2024-12-10T19:28:39,212 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 15dfd37ce6f5438f845b062fa5ab3738/cf2 of 15dfd37ce6f5438f845b062fa5ab3738 into ff9e415bfb4e4c49aa98b5a5ca8819d6(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T19:28:39,213 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 15dfd37ce6f5438f845b062fa5ab3738: 2024-12-10T19:28:39,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=15dfd37ce6f5438f845b062fa5ab3738, source=e4ad70981cbb,39803,1733858893985, destination=e4ad70981cbb,41557,1733858893868, warming up region on e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:39,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=15dfd37ce6f5438f845b062fa5ab3738, source=e4ad70981cbb,39803,1733858893985, destination=e4ad70981cbb,41557,1733858893868, running balancer 2024-12-10T19:28:39,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.HRegion(7855): Warmup {ENCODED => 15dfd37ce6f5438f845b062fa5ab3738, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:39,223 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,224 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf1 2024-12-10T19:28:39,225 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE 2024-12-10T19:28:39,229 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE 2024-12-10T19:28:39,230 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=CLOSING, regionLocation=e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:39,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE because future has completed 2024-12-10T19:28:39,240 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T19:28:39,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,39803,1733858893985}] 2024-12-10T19:28:39,249 INFO [StoreFileOpener-15dfd37ce6f5438f845b062fa5ab3738-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6139fb999dcc426ba693d2891791f485 2024-12-10T19:28:39,249 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/6139fb999dcc426ba693d2891791f485 2024-12-10T19:28:39,255 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/82da3c517cfb4855a9db02ea2712a09f 2024-12-10T19:28:39,263 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/a84360d40fd340ea8969fd7274a679b1 2024-12-10T19:28:39,263 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:39,264 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,265 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf2 2024-12-10T19:28:39,265 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-10T19:28:39,286 INFO [StoreFileOpener-15dfd37ce6f5438f845b062fa5ab3738-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26387fa911c44cb6bd4b8a6495e7370a 2024-12-10T19:28:39,286 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/26387fa911c44cb6bd4b8a6495e7370a 2024-12-10T19:28:39,297 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/ff9e415bfb4e4c49aa98b5a5ca8819d6 2024-12-10T19:28:39,297 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.HRegion(1722): Closing 15dfd37ce6f5438f845b062fa5ab3738, disabling compactions & flushes 2024-12-10T19:28:39,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. after waiting 0 ms 2024-12-10T19:28:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41557 {}] regionserver.HRegion(1676): Region close journal for 15dfd37ce6f5438f845b062fa5ab3738: Waiting for close lock at 1733858919298Disabling compacts and flushes for region at 1733858919298Disabling writes for close at 1733858919298Writing region close event to WAL at 1733858919306 (+8 ms)Closed at 1733858919306 2024-12-10T19:28:39,401 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,401 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T19:28:39,401 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 15dfd37ce6f5438f845b062fa5ab3738, disabling compactions & flushes 2024-12-10T19:28:39,401 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,401 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,402 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. after waiting 0 ms 2024-12-10T19:28:39,402 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,402 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/a84360d40fd340ea8969fd7274a679b1, hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/6139fb999dcc426ba693d2891791f485] to archive 2024-12-10T19:28:39,405 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T19:28:39,419 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/6139fb999dcc426ba693d2891791f485 to hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/6139fb999dcc426ba693d2891791f485 2024-12-10T19:28:39,419 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/a84360d40fd340ea8969fd7274a679b1 to hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/a84360d40fd340ea8969fd7274a679b1 2024-12-10T19:28:39,445 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/26387fa911c44cb6bd4b8a6495e7370a] to archive 2024-12-10T19:28:39,447 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T19:28:39,450 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/26387fa911c44cb6bd4b8a6495e7370a to hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/26387fa911c44cb6bd4b8a6495e7370a 2024-12-10T19:28:39,473 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-10T19:28:39,474 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,474 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 15dfd37ce6f5438f845b062fa5ab3738: Waiting for close lock at 1733858919401Running coprocessor pre-close hooks at 1733858919401Disabling compacts and flushes for region at 1733858919401Disabling writes for close at 1733858919402 (+1 ms)Writing region close event to WAL at 1733858919460 (+58 ms)Running coprocessor post-close hooks at 1733858919474 (+14 ms)Closed at 1733858919474 2024-12-10T19:28:39,474 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 15dfd37ce6f5438f845b062fa5ab3738 move to e4ad70981cbb,41557,1733858893868 record at close sequenceid=12 2024-12-10T19:28:39,483 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,484 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=CLOSED 2024-12-10T19:28:39,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,39803,1733858893985 because future has completed 2024-12-10T19:28:39,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-10T19:28:39,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,39803,1733858893985 in 261 msec 2024-12-10T19:28:39,513 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE; state=CLOSED, location=e4ad70981cbb,41557,1733858893868; forceNewPlan=false, retain=false 2024-12-10T19:28:39,664 INFO [e4ad70981cbb:41471 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T19:28:39,664 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=OPENING, regionLocation=e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:39,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE because future has completed 2024-12-10T19:28:39,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,41557,1733858893868}] 2024-12-10T19:28:39,844 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,844 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 15dfd37ce6f5438f845b062fa5ab3738, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:39,845 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,845 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:39,845 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,845 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,847 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,848 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf1 2024-12-10T19:28:39,849 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:39,860 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/82da3c517cfb4855a9db02ea2712a09f 2024-12-10T19:28:39,860 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:39,861 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,863 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf2 2024-12-10T19:28:39,863 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:39,880 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/ff9e415bfb4e4c49aa98b5a5ca8819d6 2024-12-10T19:28:39,880 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:39,882 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,883 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,885 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,893 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,893 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,894 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T19:28:39,901 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,902 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 15dfd37ce6f5438f845b062fa5ab3738; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60718209, jitterRate=-0.0952281802892685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T19:28:39,903 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:39,904 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 15dfd37ce6f5438f845b062fa5ab3738: Running coprocessor pre-open hook at 1733858919845Writing region info on filesystem at 1733858919845Initializing all the Stores at 1733858919846 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858919846Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858919846Cleaning up temporary data from old regions at 1733858919893 (+47 ms)Running coprocessor post-open hooks at 1733858919903 (+10 ms)Region opened successfully at 1733858919903 2024-12-10T19:28:39,911 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., pid=12, masterSystemTime=1733858919840 2024-12-10T19:28:39,916 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,916 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:39,917 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=OPEN, openSeqNum=18, regionLocation=e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:39,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,41557,1733858893868 because future has completed 2024-12-10T19:28:39,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, REOPEN/MOVE in 723 msec 2024-12-10T19:28:39,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-10T19:28:39,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,41557,1733858893868 in 245 msec 2024-12-10T19:28:39,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T19:28:39,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T19:28:39,994 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server e4ad70981cbb,41557,1733858893868: testing ***** 2024-12-10T19:28:39,994 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-10T19:28:39,997 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-10T19:28:39,998 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-10T19:28:40,005 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-10T19:28:40,007 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-10T19:28:40,025 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 506786376 }, "NonHeapMemoryUsage": { "committed": 172425216, "init": 7667712, "max": -1, "used": 169892840 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "e4ad70981cbb", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11076, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 88, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 3, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 12, "ProcessCallTime_95th_percentile": 22, "ProcessCallTime_98th_percentile": 88, "ProcessCallTime_99th_percentile": 88, "ProcessCallTime_99.9th_percentile": 88, "ProcessCallTime_TimeRangeCount_0-1": 9, "ProcessCallTime_TimeRangeCount_1-3": 9, "ProcessCallTime_TimeRangeCount_3-10": 8, "ProcessCallTime_TimeRangeCount_10-30": 4, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 4, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 2, "QueueCallTime_98th_percentile": 4, "QueueCallTime_99th_percentile": 4, "QueueCallTime_99.9th_percentile": 4, "QueueCallTime_TimeRangeCount_0-1": 29, "QueueCallTime_TimeRangeCount_1-3": 1, "QueueCallTime_TimeRangeCount_3-10": 1, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 88, "TotalCallTime_mean": 7, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 3, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 12, "TotalCallTime_95th_percentile": 24, "TotalCallTime_98th_percentile": 88, "TotalCallTime_99th_percentile": 88, "TotalCallTime_99.9th_percentile": 88, "TotalCallTime_TimeRangeCount_0-1": 7, "TotalCallTime_TimeRangeCount_1-3": 9, "TotalCallTime_TimeRangeCount_3-10": 10, "TotalCallTime_TimeRangeCount_10-30": 4, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5752 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "e4ad70981cbb", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53100", "tag.serverName": "e4ad70981cbb,41557,1733858893868", "tag.clusterId": "896f6e2b-5117-4154-b1cc-9fad4e023b6a", "tag.Context": "regionserver", "tag.Hostname": "e4ad70981cbb", "regionCount": 1, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733858893868, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 1, "ScanTime_max": 1, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 6, "Put_max": 6, "Put_mean": 6, "Put_25th_percentile": 6, "Put_median": 6, "Put_75th_percentile": 6, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_3-10": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 183, "FlushTime_max": 183, "FlushTime_mean": 183, "FlushTime_25th_percentile": 183, "FlushTime_median": 183, "FlushTime_75th_percentile": 183, "FlushTime_90th_percentile": 183, "FlushTime_95th_percentile": 183, "FlushTime_98th_percentile": 183, "FlushTime_99th_percentile": 183, "FlushTime_99.9th_percentile": 183, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-10T19:28:40,033 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41471 {}] master.MasterRpcServices(700): e4ad70981cbb,41557,1733858893868 reported a fatal error: ***** ABORTING region server e4ad70981cbb,41557,1733858893868: testing ***** 2024-12-10T19:28:40,039 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e4ad70981cbb,41557,1733858893868' ***** 2024-12-10T19:28:40,039 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-10T19:28:40,039 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T19:28:40,040 INFO [regionserver/e4ad70981cbb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T19:28:40,040 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T19:28:40,040 INFO [RS:0;e4ad70981cbb:41557 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-10T19:28:40,040 INFO [RS:0;e4ad70981cbb:41557 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-10T19:28:40,040 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(3091): Received CLOSE for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:40,040 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(956): aborting server e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:40,041 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T19:28:40,041 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 15dfd37ce6f5438f845b062fa5ab3738, disabling compactions & flushes 2024-12-10T19:28:40,041 INFO [RS:0;e4ad70981cbb:41557 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e4ad70981cbb:41557. 2024-12-10T19:28:40,041 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:40,041 DEBUG [RS:0;e4ad70981cbb:41557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T19:28:40,041 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:40,041 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. after waiting 0 ms 2024-12-10T19:28:40,041 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:55426 deadline: 1733858980041, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e4ad70981cbb port=41557 startCode=1733858893868. As of locationSeqNum=12. 2024-12-10T19:28:40,042 DEBUG [RS:0;e4ad70981cbb:41557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:28:40,042 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T19:28:40,042 DEBUG [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(1325): Online Regions={15dfd37ce6f5438f845b062fa5ab3738=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.} 2024-12-10T19:28:40,042 DEBUG [Time-limited test {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,39803,1733858893985, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,39803,1733858893985, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e4ad70981cbb port=41557 startCode=1733858893868. As of locationSeqNum=12. 2024-12-10T19:28:40,043 DEBUG [Time-limited test {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,39803,1733858893985, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e4ad70981cbb port=41557 startCode=1733858893868. As of locationSeqNum=12. 2024-12-10T19:28:40,043 DEBUG [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(1351): Waiting on 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:40,043 DEBUG [Time-limited test {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,39803,1733858893985, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e4ad70981cbb port=41557 startCode=1733858893868. As of locationSeqNum=12. 2024-12-10T19:28:40,074 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:40,074 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 15dfd37ce6f5438f845b062fa5ab3738: Waiting for close lock at 1733858920041Running coprocessor pre-close hooks at 1733858920041Disabling compacts and flushes for region at 1733858920041Disabling writes for close at 1733858920041Writing region close event to WAL at 1733858920073 (+32 ms)Running coprocessor post-close hooks at 1733858920073Closed at 1733858920074 (+1 ms) 2024-12-10T19:28:40,074 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:40,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e4ad70981cbb,41557,1733858893868 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:40,172 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e4ad70981cbb,41557,1733858893868 aborting 2024-12-10T19:28:40,172 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e4ad70981cbb,41557,1733858893868 aborting 2024-12-10T19:28:40,172 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=12 from cache 2024-12-10T19:28:40,243 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(976): stopping server e4ad70981cbb,41557,1733858893868; all regions closed. 2024-12-10T19:28:40,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741834_1010 (size=2288) 2024-12-10T19:28:40,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741834_1010 (size=2288) 2024-12-10T19:28:40,265 DEBUG [RS:0;e4ad70981cbb:41557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:28:40,265 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T19:28:40,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741834_1010 (size=2288) 2024-12-10T19:28:40,265 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T19:28:40,266 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.ChoreService(370): Chore service for: regionserver/e4ad70981cbb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T19:28:40,266 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T19:28:40,266 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T19:28:40,266 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T19:28:40,266 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T19:28:40,266 INFO [regionserver/e4ad70981cbb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T19:28:40,267 INFO [RS:0;e4ad70981cbb:41557 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41557 2024-12-10T19:28:40,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T19:28:40,322 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T19:28:40,322 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e4ad70981cbb,41557,1733858893868 2024-12-10T19:28:40,333 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e4ad70981cbb,41557,1733858893868] 2024-12-10T19:28:40,356 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e4ad70981cbb,41557,1733858893868 already deleted, retry=false 2024-12-10T19:28:40,357 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e4ad70981cbb,41557,1733858893868 on e4ad70981cbb,41471,1733858893036 2024-12-10T19:28:40,376 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e4ad70981cbb,41557,1733858893868, splitWal=true, meta=false 2024-12-10T19:28:40,382 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for e4ad70981cbb,41557,1733858893868 (carryingMeta=false) e4ad70981cbb,41557,1733858893868/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@789ea31[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-10T19:28:40,384 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18] 2024-12-10T19:28:40,387 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e4ad70981cbb:41557 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e4ad70981cbb/172.17.0.3:41557 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:40,388 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18, error=java.net.ConnectException: Call to address=e4ad70981cbb:41557 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e4ad70981cbb/172.17.0.3:41557 2024-12-10T19:28:40,388 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-10T19:28:40,388 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18 from cache 2024-12-10T19:28:40,388 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address e4ad70981cbb:41557 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e4ad70981cbb/172.17.0.3:41557 2024-12-10T19:28:40,391 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e4ad70981cbb,41557,1733858893868, splitWal=true, meta=false 2024-12-10T19:28:40,398 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): e4ad70981cbb,41557,1733858893868 had 1 regions 2024-12-10T19:28:40,401 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure e4ad70981cbb,41557,1733858893868, splitWal=true, meta=false, isMeta: false 2024-12-10T19:28:40,420 DEBUG [PEWorker-3 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting 2024-12-10T19:28:40,429 INFO [PEWorker-3 {}] master.SplitWALManager(105): e4ad70981cbb,41557,1733858893868 WAL count=1, meta=false 2024-12-10T19:28:40,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e4ad70981cbb%2C41557%2C1733858893868.1733858896435}] 2024-12-10T19:28:40,437 INFO [RS:0;e4ad70981cbb:41557 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T19:28:40,437 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T19:28:40,437 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41557-0x100114e4b300001, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T19:28:40,438 INFO [RS:0;e4ad70981cbb:41557 {}] regionserver.HRegionServer(1031): Exiting; stopping=e4ad70981cbb,41557,1733858893868; zookeeper connection closed. 2024-12-10T19:28:40,438 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ede6298 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ede6298 2024-12-10T19:28:40,462 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:40,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e4ad70981cbb%2C41557%2C1733858893868.1733858896435, worker=e4ad70981cbb,39803,1733858893985}] 2024-12-10T19:28:40,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39803 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-10T19:28:40,703 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18] 2024-12-10T19:28:40,704 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to e4ad70981cbb:41557 this server is in the failed servers list 2024-12-10T19:28:40,705 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=e4ad70981cbb:41557 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e4ad70981cbb:41557 2024-12-10T19:28:40,705 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e4ad70981cbb:41557 2024-12-10T19:28:40,705 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18 from cache 2024-12-10T19:28:40,706 INFO [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435, size=2.2 K (2288bytes) 2024-12-10T19:28:40,706 INFO [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435 2024-12-10T19:28:40,709 INFO [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435 after 3ms 2024-12-10T19:28:40,775 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:40,776 INFO [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435 took 70ms 2024-12-10T19:28:40,810 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 15dfd37ce6f5438f845b062fa5ab3738: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-10T19:28:40,810 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435 so closing down 2024-12-10T19:28:40,810 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:40,816 INFO [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:40,816 INFO [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 28 ms; skipped=6; WAL=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-10T19:28:40,817 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435, journal: Splitting hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435, size=2.2 K (2288bytes) at 1733858920706Finishing writing output for hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435 so closing down at 1733858920810 (+104 ms)3 split writer threads finished at 1733858920816 (+6 ms)Processed 6 edits across 0 Regions in 28 ms; skipped=6; WAL=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733858920816 2024-12-10T19:28:40,817 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435 2024-12-10T19:28:40,819 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e4ad70981cbb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-10T19:28:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-10T19:28:40,860 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting/e4ad70981cbb%2C41557%2C1733858893868.1733858896435 to hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/oldWALs 2024-12-10T19:28:40,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-10T19:28:40,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e4ad70981cbb%2C41557%2C1733858893868.1733858896435, worker=e4ad70981cbb,39803,1733858893985 in 387 msec 2024-12-10T19:28:40,894 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=e4ad70981cbb,39803,1733858893985 2024-12-10T19:28:40,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-10T19:28:40,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure e4ad70981cbb%2C41557%2C1733858893868.1733858896435, worker=e4ad70981cbb,39803,1733858893985 in 463 msec 2024-12-10T19:28:40,925 INFO [PEWorker-3 {}] master.SplitLogManager(171): hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting dir is empty, no logs to split. 2024-12-10T19:28:40,925 INFO [PEWorker-3 {}] master.SplitWALManager(105): e4ad70981cbb,41557,1733858893868 WAL count=0, meta=false 2024-12-10T19:28:40,925 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if e4ad70981cbb,41557,1733858893868 WAL splitting is done? wals=0, meta=false 2024-12-10T19:28:40,928 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for e4ad70981cbb,41557,1733858893868 failed, ignore...File hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,41557,1733858893868-splitting does not exist. 2024-12-10T19:28:40,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN}] 2024-12-10T19:28:40,934 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN 2024-12-10T19:28:40,936 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T19:28:41,087 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(204): Hosts are {e4ad70981cbb=0} racks are {/default-rack=0} 2024-12-10T19:28:41,088 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T19:28:41,088 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T19:28:41,088 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T19:28:41,088 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T19:28:41,088 INFO [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T19:28:41,088 INFO [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T19:28:41,088 DEBUG [e4ad70981cbb:41471 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T19:28:41,088 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=OPENING, regionLocation=e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:41,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN because future has completed 2024-12-10T19:28:41,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,35427,1733858894073}] 2024-12-10T19:28:41,231 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18] 2024-12-10T19:28:41,231 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to e4ad70981cbb:41557 this server is in the failed servers list 2024-12-10T19:28:41,232 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=e4ad70981cbb:41557 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e4ad70981cbb:41557 2024-12-10T19:28:41,232 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e4ad70981cbb:41557 2024-12-10T19:28:41,232 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,41557,1733858893868, seqNum=18 from cache 2024-12-10T19:28:41,327 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:41,327 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 15dfd37ce6f5438f845b062fa5ab3738, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:41,328 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,328 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:41,328 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,328 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,344 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,349 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf1 2024-12-10T19:28:41,349 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:41,420 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf1/82da3c517cfb4855a9db02ea2712a09f 2024-12-10T19:28:41,420 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:41,421 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,441 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15dfd37ce6f5438f845b062fa5ab3738 columnFamilyName cf2 2024-12-10T19:28:41,441 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:41,470 DEBUG [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/cf2/ff9e415bfb4e4c49aa98b5a5ca8819d6 2024-12-10T19:28:41,470 INFO [StoreOpener-15dfd37ce6f5438f845b062fa5ab3738-1 {}] regionserver.HStore(327): Store=15dfd37ce6f5438f845b062fa5ab3738/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:41,473 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,474 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,476 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,478 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,478 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,479 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T19:28:41,482 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,483 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 15dfd37ce6f5438f845b062fa5ab3738; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75098505, jitterRate=0.11905492842197418}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T19:28:41,484 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:28:41,484 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 15dfd37ce6f5438f845b062fa5ab3738: Running coprocessor pre-open hook at 1733858921328Writing region info on filesystem at 1733858921328Initializing all the Stores at 1733858921337 (+9 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858921337Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858921340 (+3 ms)Cleaning up temporary data from old regions at 1733858921478 (+138 ms)Running coprocessor post-open hooks at 1733858921484 (+6 ms)Region opened successfully at 1733858921484 2024-12-10T19:28:41,495 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., pid=17, masterSystemTime=1733858921280 2024-12-10T19:28:41,506 DEBUG [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:41,506 INFO [RS_OPEN_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:28:41,506 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=15dfd37ce6f5438f845b062fa5ab3738, regionState=OPEN, openSeqNum=18, regionLocation=e4ad70981cbb,35427,1733858894073 2024-12-10T19:28:41,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,35427,1733858894073 because future has completed 2024-12-10T19:28:41,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-10T19:28:41,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 15dfd37ce6f5438f845b062fa5ab3738, server=e4ad70981cbb,35427,1733858894073 in 403 msec 2024-12-10T19:28:41,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-10T19:28:41,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=15dfd37ce6f5438f845b062fa5ab3738, ASSIGN in 597 msec 2024-12-10T19:28:41,546 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server e4ad70981cbb,41557,1733858893868 after splitting done 2024-12-10T19:28:41,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure e4ad70981cbb,41557,1733858893868, splitWal=true, meta=false in 1.1880 sec 2024-12-10T19:28:42,245 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738., hostname=e4ad70981cbb,35427,1733858894073, seqNum=18] 2024-12-10T19:28:42,277 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 412), OpenFileDescriptor=1043 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=700 (was 682) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3836 (was 2615) - AvailableMemoryMB LEAK? - 2024-12-10T19:28:42,279 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1043 is superior to 1024 2024-12-10T19:28:42,300 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1043, MaxFileDescriptor=1048576, SystemLoadAverage=700, ProcessCount=11, AvailableMemoryMB=3835 2024-12-10T19:28:42,300 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1043 is superior to 1024 2024-12-10T19:28:42,326 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:42,329 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:42,331 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:42,335 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23014700, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-23014700, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:42,355 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23014700/hregion-23014700.1733858922335, exclude list is [], retry=0 2024-12-10T19:28:42,380 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:42,393 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:42,409 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:42,470 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23014700/hregion-23014700.1733858922335 2024-12-10T19:28:42,471 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:42,471 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f85f4c449a9f64d054f08ba203b5c03a, NAME => 'testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:28:42,525 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T19:28:42,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741893_1071 (size=67) 2024-12-10T19:28:42,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741893_1071 (size=67) 2024-12-10T19:28:42,537 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:42,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741893_1071 (size=67) 2024-12-10T19:28:42,545 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,551 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85f4c449a9f64d054f08ba203b5c03a columnFamilyName a 2024-12-10T19:28:42,552 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:42,553 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(327): Store=f85f4c449a9f64d054f08ba203b5c03a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:42,554 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,556 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85f4c449a9f64d054f08ba203b5c03a columnFamilyName b 2024-12-10T19:28:42,556 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:42,557 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(327): Store=f85f4c449a9f64d054f08ba203b5c03a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:42,558 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,569 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85f4c449a9f64d054f08ba203b5c03a columnFamilyName c 2024-12-10T19:28:42,569 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:42,570 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(327): Store=f85f4c449a9f64d054f08ba203b5c03a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:42,570 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,572 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,573 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,581 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,581 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,584 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:42,586 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,602 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:42,607 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f85f4c449a9f64d054f08ba203b5c03a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66697530, jitterRate=-0.006129354238510132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:42,613 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f85f4c449a9f64d054f08ba203b5c03a: Writing region info on filesystem at 1733858922537Initializing all the Stores at 1733858922543 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858922543Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858922544 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858922544Cleaning up temporary data from old regions at 1733858922581 (+37 ms)Region opened successfully at 1733858922613 (+32 ms) 2024-12-10T19:28:42,613 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f85f4c449a9f64d054f08ba203b5c03a, disabling compactions & flushes 2024-12-10T19:28:42,613 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. 2024-12-10T19:28:42,613 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. 2024-12-10T19:28:42,613 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. after waiting 0 ms 2024-12-10T19:28:42,613 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. 2024-12-10T19:28:42,619 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. 2024-12-10T19:28:42,620 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f85f4c449a9f64d054f08ba203b5c03a: Waiting for close lock at 1733858922613Disabling compacts and flushes for region at 1733858922613Disabling writes for close at 1733858922613Writing region close event to WAL at 1733858922619 (+6 ms)Closed at 1733858922619 2024-12-10T19:28:42,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741892_1070 (size=175) 2024-12-10T19:28:42,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741892_1070 (size=175) 2024-12-10T19:28:42,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741892_1070 (size=175) 2024-12-10T19:28:42,669 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:42,669 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23014700:(num 1733858922335) 2024-12-10T19:28:42,669 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:42,672 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:42,700 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672, exclude list is [], retry=0 2024-12-10T19:28:42,725 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:42,725 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:42,725 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:42,853 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672 2024-12-10T19:28:42,857 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:28:42,857 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f85f4c449a9f64d054f08ba203b5c03a, NAME => 'testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:42,858 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:42,858 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,858 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,860 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,861 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85f4c449a9f64d054f08ba203b5c03a columnFamilyName a 2024-12-10T19:28:42,861 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:42,862 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(327): Store=f85f4c449a9f64d054f08ba203b5c03a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:42,862 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,863 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85f4c449a9f64d054f08ba203b5c03a columnFamilyName b 2024-12-10T19:28:42,864 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:42,865 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(327): Store=f85f4c449a9f64d054f08ba203b5c03a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:42,865 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,870 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85f4c449a9f64d054f08ba203b5c03a columnFamilyName c 2024-12-10T19:28:42,870 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:42,875 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(327): Store=f85f4c449a9f64d054f08ba203b5c03a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:42,875 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,885 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,892 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,898 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,898 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,899 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:42,910 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:42,912 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f85f4c449a9f64d054f08ba203b5c03a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69296388, jitterRate=0.0325966477394104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:42,913 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f85f4c449a9f64d054f08ba203b5c03a: Writing region info on filesystem at 1733858922858Initializing all the Stores at 1733858922859 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858922859Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858922860 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858922860Cleaning up temporary data from old regions at 1733858922898 (+38 ms)Region opened successfully at 1733858922913 (+15 ms) 2024-12-10T19:28:42,994 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T19:28:43,033 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f85f4c449a9f64d054f08ba203b5c03a 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-10T19:28:43,072 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/a/c2f327621f2940bbb817335425c81376 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733858922913/Put/seqid=0 2024-12-10T19:28:43,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741895_1073 (size=5958) 2024-12-10T19:28:43,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741895_1073 (size=5958) 2024-12-10T19:28:43,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741895_1073 (size=5958) 2024-12-10T19:28:43,159 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/a/c2f327621f2940bbb817335425c81376 2024-12-10T19:28:43,238 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/b/227d24e453e643909dd53ac6ee93acb8 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733858922935/Put/seqid=0 2024-12-10T19:28:43,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741896_1074 (size=5958) 2024-12-10T19:28:43,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741896_1074 (size=5958) 2024-12-10T19:28:43,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741896_1074 (size=5958) 2024-12-10T19:28:43,263 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/b/227d24e453e643909dd53ac6ee93acb8 2024-12-10T19:28:43,304 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/c/3bd5d7dce5d747c3b4c7cb13a1a29949 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733858922990/Put/seqid=0 2024-12-10T19:28:43,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741897_1075 (size=5958) 2024-12-10T19:28:43,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741897_1075 (size=5958) 2024-12-10T19:28:43,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741897_1075 (size=5958) 2024-12-10T19:28:43,367 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/c/3bd5d7dce5d747c3b4c7cb13a1a29949 2024-12-10T19:28:43,389 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/a/c2f327621f2940bbb817335425c81376 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/a/c2f327621f2940bbb817335425c81376 2024-12-10T19:28:43,407 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/a/c2f327621f2940bbb817335425c81376, entries=10, sequenceid=33, filesize=5.8 K 2024-12-10T19:28:43,411 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/b/227d24e453e643909dd53ac6ee93acb8 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/b/227d24e453e643909dd53ac6ee93acb8 2024-12-10T19:28:43,428 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/b/227d24e453e643909dd53ac6ee93acb8, entries=10, sequenceid=33, filesize=5.8 K 2024-12-10T19:28:43,429 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/c/3bd5d7dce5d747c3b4c7cb13a1a29949 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/c/3bd5d7dce5d747c3b4c7cb13a1a29949 2024-12-10T19:28:43,449 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/c/3bd5d7dce5d747c3b4c7cb13a1a29949, entries=10, sequenceid=33, filesize=5.8 K 2024-12-10T19:28:43,452 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-10T19:28:43,452 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-10T19:28:43,454 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-10T19:28:43,454 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-10T19:28:43,457 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f85f4c449a9f64d054f08ba203b5c03a in 424ms, sequenceid=33, compaction requested=false 2024-12-10T19:28:43,457 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f85f4c449a9f64d054f08ba203b5c03a: 2024-12-10T19:28:43,457 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f85f4c449a9f64d054f08ba203b5c03a, disabling compactions & flushes 2024-12-10T19:28:43,457 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. 2024-12-10T19:28:43,457 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. 2024-12-10T19:28:43,457 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. after waiting 0 ms 2024-12-10T19:28:43,457 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. 2024-12-10T19:28:43,499 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. 2024-12-10T19:28:43,499 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f85f4c449a9f64d054f08ba203b5c03a: Waiting for close lock at 1733858923457Disabling compacts and flushes for region at 1733858923457Disabling writes for close at 1733858923457Writing region close event to WAL at 1733858923499 (+42 ms)Closed at 1733858923499 2024-12-10T19:28:43,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741894_1072 (size=6311) 2024-12-10T19:28:43,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741894_1072 (size=6311) 2024-12-10T19:28:43,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741894_1072 (size=6311) 2024-12-10T19:28:43,552 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/b/227d24e453e643909dd53ac6ee93acb8 to hdfs://localhost:46813/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/b/227d24e453e643909dd53ac6ee93acb8 2024-12-10T19:28:43,592 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672, size=6.2 K (6311bytes) 2024-12-10T19:28:43,592 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672 2024-12-10T19:28:43,598 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672 after 6ms 2024-12-10T19:28:43,667 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:43,667 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672 took 76ms 2024-12-10T19:28:43,703 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672 so closing down 2024-12-10T19:28:43,703 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:43,728 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733858922672.temp 2024-12-10T19:28:43,785 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000003-wal.1733858922672.temp 2024-12-10T19:28:43,785 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:43,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741898_1076 (size=5575) 2024-12-10T19:28:43,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741898_1076 (size=5575) 2024-12-10T19:28:43,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741898_1076 (size=5575) 2024-12-10T19:28:43,856 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000003-wal.1733858922672.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-10T19:28:43,864 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000003-wal.1733858922672.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000032 2024-12-10T19:28:43,865 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 169 ms; skipped=2; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-10T19:28:43,865 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672, journal: Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672, size=6.2 K (6311bytes) at 1733858923592Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672 so closing down at 1733858923703 (+111 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000003-wal.1733858922672.temp at 1733858923785 (+82 ms)3 split writer threads finished at 1733858923785Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000003-wal.1733858922672.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733858923856 (+71 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000003-wal.1733858922672.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000032 at 1733858923864 (+8 ms)Processed 32 edits across 1 Regions in 169 ms; skipped=2; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733858923865 (+1 ms) 2024-12-10T19:28:43,868 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858922672 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858922672 2024-12-10T19:28:43,870 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000032 2024-12-10T19:28:43,870 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:43,873 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:43,902 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858923874, exclude list is [], retry=0 2024-12-10T19:28:43,908 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:43,909 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:43,912 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:44,029 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858923874 2024-12-10T19:28:44,040 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:28:44,041 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f85f4c449a9f64d054f08ba203b5c03a, NAME => 'testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:44,041 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:44,041 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,041 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,072 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,077 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85f4c449a9f64d054f08ba203b5c03a columnFamilyName a 2024-12-10T19:28:44,077 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:44,098 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/a/c2f327621f2940bbb817335425c81376 2024-12-10T19:28:44,098 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(327): Store=f85f4c449a9f64d054f08ba203b5c03a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:44,098 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,101 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85f4c449a9f64d054f08ba203b5c03a columnFamilyName b 2024-12-10T19:28:44,101 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:44,102 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(327): Store=f85f4c449a9f64d054f08ba203b5c03a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:44,102 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,104 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85f4c449a9f64d054f08ba203b5c03a columnFamilyName c 2024-12-10T19:28:44,104 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:44,149 DEBUG [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/c/3bd5d7dce5d747c3b4c7cb13a1a29949 2024-12-10T19:28:44,149 INFO [StoreOpener-f85f4c449a9f64d054f08ba203b5c03a-1 {}] regionserver.HStore(327): Store=f85f4c449a9f64d054f08ba203b5c03a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:44,150 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,155 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,160 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000032 2024-12-10T19:28:44,227 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:44,235 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000032 2024-12-10T19:28:44,235 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f85f4c449a9f64d054f08ba203b5c03a 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-10T19:28:44,291 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/b/bdf991c7eb974d919f5c2a148dfa84f6 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733858922935/Put/seqid=0 2024-12-10T19:28:44,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741900_1078 (size=5958) 2024-12-10T19:28:44,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741900_1078 (size=5958) 2024-12-10T19:28:44,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741900_1078 (size=5958) 2024-12-10T19:28:44,393 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/b/bdf991c7eb974d919f5c2a148dfa84f6 2024-12-10T19:28:44,432 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/.tmp/b/bdf991c7eb974d919f5c2a148dfa84f6 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/b/bdf991c7eb974d919f5c2a148dfa84f6 2024-12-10T19:28:44,467 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/b/bdf991c7eb974d919f5c2a148dfa84f6, entries=10, sequenceid=32, filesize=5.8 K 2024-12-10T19:28:44,468 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f85f4c449a9f64d054f08ba203b5c03a in 233ms, sequenceid=32, compaction requested=false; wal=null 2024-12-10T19:28:44,471 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/0000000000000000032 2024-12-10T19:28:44,474 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,474 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,478 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:44,483 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f85f4c449a9f64d054f08ba203b5c03a 2024-12-10T19:28:44,520 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/f85f4c449a9f64d054f08ba203b5c03a/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-10T19:28:44,523 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f85f4c449a9f64d054f08ba203b5c03a; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68809720, jitterRate=0.02534472942352295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:44,524 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f85f4c449a9f64d054f08ba203b5c03a: Writing region info on filesystem at 1733858924041Initializing all the Stores at 1733858924048 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858924048Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858924063 (+15 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858924063Obtaining lock to block concurrent updates at 1733858924235 (+172 ms)Preparing flush snapshotting stores in f85f4c449a9f64d054f08ba203b5c03a at 1733858924235Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733858924236 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733858922327.f85f4c449a9f64d054f08ba203b5c03a. at 1733858924236Flushing f85f4c449a9f64d054f08ba203b5c03a/b: creating writer at 1733858924236Flushing f85f4c449a9f64d054f08ba203b5c03a/b: appending metadata at 1733858924290 (+54 ms)Flushing f85f4c449a9f64d054f08ba203b5c03a/b: closing flushed file at 1733858924290Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@711dff10: reopening flushed file at 1733858924418 (+128 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f85f4c449a9f64d054f08ba203b5c03a in 233ms, sequenceid=32, compaction requested=false; wal=null at 1733858924468 (+50 ms)Cleaning up temporary data from old regions at 1733858924474 (+6 ms)Region opened successfully at 1733858924523 (+49 ms) 2024-12-10T19:28:44,570 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=419 (was 410) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:38208 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:54272 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:41490 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:40206 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:54304 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:54588 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1114 (was 1043) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=700 (was 700), ProcessCount=11 (was 11), AvailableMemoryMB=3609 (was 3835) 2024-12-10T19:28:44,570 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1114 is superior to 1024 2024-12-10T19:28:44,585 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1114, MaxFileDescriptor=1048576, SystemLoadAverage=700, ProcessCount=11, AvailableMemoryMB=3606 2024-12-10T19:28:44,585 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1114 is superior to 1024 2024-12-10T19:28:44,608 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:44,626 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:44,626 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:44,634 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-31090794, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-31090794, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:44,653 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-31090794/hregion-31090794.1733858924635, exclude list is [], retry=0 2024-12-10T19:28:44,663 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:44,666 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:44,669 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:44,728 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-31090794/hregion-31090794.1733858924635 2024-12-10T19:28:44,733 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:44,733 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5abbba8deb8d6be0f8eb007189505b94, NAME => 'testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:28:44,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741902_1080 (size=68) 2024-12-10T19:28:44,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741902_1080 (size=68) 2024-12-10T19:28:44,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741902_1080 (size=68) 2024-12-10T19:28:45,260 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:45,261 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,264 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5abbba8deb8d6be0f8eb007189505b94 columnFamilyName a 2024-12-10T19:28:45,264 DEBUG [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:45,265 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(327): Store=5abbba8deb8d6be0f8eb007189505b94/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:45,265 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,272 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5abbba8deb8d6be0f8eb007189505b94 columnFamilyName b 2024-12-10T19:28:45,272 DEBUG [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:45,273 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(327): Store=5abbba8deb8d6be0f8eb007189505b94/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:45,273 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,280 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5abbba8deb8d6be0f8eb007189505b94 columnFamilyName c 2024-12-10T19:28:45,280 DEBUG [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:45,283 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(327): Store=5abbba8deb8d6be0f8eb007189505b94/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:45,283 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,284 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,285 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,285 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,286 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,286 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:45,287 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,293 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:45,294 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5abbba8deb8d6be0f8eb007189505b94; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74871849, jitterRate=0.11567749083042145}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:45,295 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5abbba8deb8d6be0f8eb007189505b94: Writing region info on filesystem at 1733858925260Initializing all the Stores at 1733858925261 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858925261Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858925261Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858925261Cleaning up temporary data from old regions at 1733858925286 (+25 ms)Region opened successfully at 1733858925294 (+8 ms) 2024-12-10T19:28:45,295 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5abbba8deb8d6be0f8eb007189505b94, disabling compactions & flushes 2024-12-10T19:28:45,295 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:45,295 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:45,295 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. after waiting 0 ms 2024-12-10T19:28:45,295 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:45,312 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:45,312 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5abbba8deb8d6be0f8eb007189505b94: Waiting for close lock at 1733858925295Disabling compacts and flushes for region at 1733858925295Disabling writes for close at 1733858925295Writing region close event to WAL at 1733858925312 (+17 ms)Closed at 1733858925312 2024-12-10T19:28:45,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741901_1079 (size=175) 2024-12-10T19:28:45,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741901_1079 (size=175) 2024-12-10T19:28:45,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741901_1079 (size=175) 2024-12-10T19:28:45,368 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:45,368 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-31090794:(num 1733858924635) 2024-12-10T19:28:45,369 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:45,412 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:45,432 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413, exclude list is [], retry=0 2024-12-10T19:28:45,434 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:45,435 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:45,435 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:45,466 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413 2024-12-10T19:28:45,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:45,550 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5abbba8deb8d6be0f8eb007189505b94, NAME => 'testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:45,552 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,552 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:45,553 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,553 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,555 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,556 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5abbba8deb8d6be0f8eb007189505b94 columnFamilyName a 2024-12-10T19:28:45,556 DEBUG [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:45,557 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(327): Store=5abbba8deb8d6be0f8eb007189505b94/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:45,558 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,559 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5abbba8deb8d6be0f8eb007189505b94 columnFamilyName b 2024-12-10T19:28:45,559 DEBUG [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:45,561 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(327): Store=5abbba8deb8d6be0f8eb007189505b94/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:45,561 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,563 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5abbba8deb8d6be0f8eb007189505b94 columnFamilyName c 2024-12-10T19:28:45,564 DEBUG [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:45,565 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(327): Store=5abbba8deb8d6be0f8eb007189505b94/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:45,565 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,566 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,569 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,569 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,570 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:45,573 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,575 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5abbba8deb8d6be0f8eb007189505b94; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62093017, jitterRate=-0.07474194467067719}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:45,575 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:45,576 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5abbba8deb8d6be0f8eb007189505b94: Running coprocessor pre-open hook at 1733858925553Writing region info on filesystem at 1733858925553Initializing all the Stores at 1733858925554 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858925554Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858925554Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858925554Cleaning up temporary data from old regions at 1733858925569 (+15 ms)Running coprocessor post-open hooks at 1733858925575 (+6 ms)Region opened successfully at 1733858925576 (+1 ms) 2024-12-10T19:28:45,606 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5abbba8deb8d6be0f8eb007189505b94 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-10T19:28:45,608 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:46,609 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:47,610 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:48,611 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:49,612 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:50,613 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:51,055 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T19:28:51,616 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:52,617 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:53,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-10T19:28:53,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-10T19:28:53,618 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:54,619 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5abbba8deb8d6be0f8eb007189505b94/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:54,620 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5abbba8deb8d6be0f8eb007189505b94: 2024-12-10T19:28:54,620 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:54,653 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5abbba8deb8d6be0f8eb007189505b94: 2024-12-10T19:28:54,653 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-10T19:28:54,653 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5abbba8deb8d6be0f8eb007189505b94, disabling compactions & flushes 2024-12-10T19:28:54,653 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:54,653 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:54,653 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. after waiting 0 ms 2024-12-10T19:28:54,653 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:54,654 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:54,654 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. 2024-12-10T19:28:54,654 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5abbba8deb8d6be0f8eb007189505b94: Waiting for close lock at 1733858934653Running coprocessor pre-close hooks at 1733858934653Disabling compacts and flushes for region at 1733858934653Disabling writes for close at 1733858934653Writing region close event to WAL at 1733858934653Running coprocessor post-close hooks at 1733858934654 (+1 ms)Closed at 1733858934654 2024-12-10T19:28:54,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741903_1081 (size=4236) 2024-12-10T19:28:54,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741903_1081 (size=4236) 2024-12-10T19:28:54,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741903_1081 (size=4236) 2024-12-10T19:28:54,691 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413, size=4.1 K (4236bytes) 2024-12-10T19:28:54,691 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413 2024-12-10T19:28:54,693 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413 after 2ms 2024-12-10T19:28:54,741 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:54,742 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413 took 51ms 2024-12-10T19:28:54,746 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413 so closing down 2024-12-10T19:28:54,746 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:54,751 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733858925413.temp 2024-12-10T19:28:54,789 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000004-wal.1733858925413.temp 2024-12-10T19:28:54,790 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:54,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741904_1082 (size=3245) 2024-12-10T19:28:54,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741904_1082 (size=3245) 2024-12-10T19:28:54,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741904_1082 (size=3245) 2024-12-10T19:28:54,808 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000004-wal.1733858925413.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-10T19:28:54,809 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000004-wal.1733858925413.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000026 2024-12-10T19:28:54,810 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 67 ms; skipped=3; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-10T19:28:54,810 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413, journal: Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413, size=4.1 K (4236bytes) at 1733858934691Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413 so closing down at 1733858934746 (+55 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000004-wal.1733858925413.temp at 1733858934789 (+43 ms)3 split writer threads finished at 1733858934790 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000004-wal.1733858925413.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733858934808 (+18 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000004-wal.1733858925413.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000026 at 1733858934809 (+1 ms)Processed 23 edits across 1 Regions in 67 ms; skipped=3; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733858934810 (+1 ms) 2024-12-10T19:28:54,812 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858925413 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858925413 2024-12-10T19:28:54,813 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000026 2024-12-10T19:28:54,813 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:54,815 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:54,827 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858934816, exclude list is [], retry=0 2024-12-10T19:28:54,830 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:54,831 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:54,831 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:54,878 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858934816 2024-12-10T19:28:54,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:54,879 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5abbba8deb8d6be0f8eb007189505b94, NAME => 'testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:54,880 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:54,880 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:54,880 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:54,880 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:54,883 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:54,885 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5abbba8deb8d6be0f8eb007189505b94 columnFamilyName a 2024-12-10T19:28:54,885 DEBUG [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:54,887 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(327): Store=5abbba8deb8d6be0f8eb007189505b94/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:54,887 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:54,889 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5abbba8deb8d6be0f8eb007189505b94 columnFamilyName b 2024-12-10T19:28:54,889 DEBUG [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:54,890 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(327): Store=5abbba8deb8d6be0f8eb007189505b94/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:54,890 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:54,891 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5abbba8deb8d6be0f8eb007189505b94 columnFamilyName c 2024-12-10T19:28:54,891 DEBUG [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:54,892 INFO [StoreOpener-5abbba8deb8d6be0f8eb007189505b94-1 {}] regionserver.HStore(327): Store=5abbba8deb8d6be0f8eb007189505b94/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:54,892 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:54,893 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:54,896 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:54,897 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000026 2024-12-10T19:28:54,935 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:54,938 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000026 2024-12-10T19:28:54,939 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5abbba8deb8d6be0f8eb007189505b94 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-10T19:28:54,977 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/.tmp/a/22c3b49718844856ac0c5dbf7901e1e1 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733858934629/Put/seqid=0 2024-12-10T19:28:54,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741906_1084 (size=5523) 2024-12-10T19:28:54,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741906_1084 (size=5523) 2024-12-10T19:28:54,996 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/.tmp/a/22c3b49718844856ac0c5dbf7901e1e1 2024-12-10T19:28:54,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741906_1084 (size=5523) 2024-12-10T19:28:55,038 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/.tmp/b/21bd31a0614842c4bf2c4ba4eb773597 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733858934622/Put/seqid=0 2024-12-10T19:28:55,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741907_1085 (size=5524) 2024-12-10T19:28:55,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741907_1085 (size=5524) 2024-12-10T19:28:55,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741907_1085 (size=5524) 2024-12-10T19:28:55,047 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/.tmp/b/21bd31a0614842c4bf2c4ba4eb773597 2024-12-10T19:28:55,071 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/.tmp/c/224880ca37ed4ce98413ce1854e35b8c is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733858934626/Put/seqid=0 2024-12-10T19:28:55,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741908_1086 (size=5457) 2024-12-10T19:28:55,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741908_1086 (size=5457) 2024-12-10T19:28:55,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741908_1086 (size=5457) 2024-12-10T19:28:55,087 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/.tmp/c/224880ca37ed4ce98413ce1854e35b8c 2024-12-10T19:28:55,095 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/.tmp/a/22c3b49718844856ac0c5dbf7901e1e1 as hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/a/22c3b49718844856ac0c5dbf7901e1e1 2024-12-10T19:28:55,102 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/a/22c3b49718844856ac0c5dbf7901e1e1, entries=7, sequenceid=26, filesize=5.4 K 2024-12-10T19:28:55,103 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/.tmp/b/21bd31a0614842c4bf2c4ba4eb773597 as hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/b/21bd31a0614842c4bf2c4ba4eb773597 2024-12-10T19:28:55,109 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/b/21bd31a0614842c4bf2c4ba4eb773597, entries=7, sequenceid=26, filesize=5.4 K 2024-12-10T19:28:55,110 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/.tmp/c/224880ca37ed4ce98413ce1854e35b8c as hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/c/224880ca37ed4ce98413ce1854e35b8c 2024-12-10T19:28:55,116 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/c/224880ca37ed4ce98413ce1854e35b8c, entries=6, sequenceid=26, filesize=5.3 K 2024-12-10T19:28:55,117 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 5abbba8deb8d6be0f8eb007189505b94 in 178ms, sequenceid=26, compaction requested=false; wal=null 2024-12-10T19:28:55,118 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/0000000000000000026 2024-12-10T19:28:55,120 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:55,120 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:55,130 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:55,132 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:55,135 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsAfterAbortingFlush/5abbba8deb8d6be0f8eb007189505b94/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-10T19:28:55,137 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5abbba8deb8d6be0f8eb007189505b94; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73840732, jitterRate=0.10031265020370483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:55,137 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5abbba8deb8d6be0f8eb007189505b94 2024-12-10T19:28:55,138 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5abbba8deb8d6be0f8eb007189505b94: Running coprocessor pre-open hook at 1733858934880Writing region info on filesystem at 1733858934880Initializing all the Stores at 1733858934882 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858934882Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858934883 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858934883Obtaining lock to block concurrent updates at 1733858934939 (+56 ms)Preparing flush snapshotting stores in 5abbba8deb8d6be0f8eb007189505b94 at 1733858934939Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733858934939Flushing stores of testReplayEditsAfterAbortingFlush,,1733858924620.5abbba8deb8d6be0f8eb007189505b94. at 1733858934939Flushing 5abbba8deb8d6be0f8eb007189505b94/a: creating writer at 1733858934940 (+1 ms)Flushing 5abbba8deb8d6be0f8eb007189505b94/a: appending metadata at 1733858934976 (+36 ms)Flushing 5abbba8deb8d6be0f8eb007189505b94/a: closing flushed file at 1733858934976Flushing 5abbba8deb8d6be0f8eb007189505b94/b: creating writer at 1733858935001 (+25 ms)Flushing 5abbba8deb8d6be0f8eb007189505b94/b: appending metadata at 1733858935038 (+37 ms)Flushing 5abbba8deb8d6be0f8eb007189505b94/b: closing flushed file at 1733858935038Flushing 5abbba8deb8d6be0f8eb007189505b94/c: creating writer at 1733858935055 (+17 ms)Flushing 5abbba8deb8d6be0f8eb007189505b94/c: appending metadata at 1733858935070 (+15 ms)Flushing 5abbba8deb8d6be0f8eb007189505b94/c: closing flushed file at 1733858935070Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a1a5eaa: reopening flushed file at 1733858935093 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2feea9db: reopening flushed file at 1733858935102 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bd3bce0: reopening flushed file at 1733858935109 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 5abbba8deb8d6be0f8eb007189505b94 in 178ms, sequenceid=26, compaction requested=false; wal=null at 1733858935117 (+8 ms)Cleaning up temporary data from old regions at 1733858935120 (+3 ms)Running coprocessor post-open hooks at 1733858935137 (+17 ms)Region opened successfully at 1733858935138 (+1 ms) 2024-12-10T19:28:55,159 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=425 (was 419) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:49126 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34870 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:38250 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34896 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:49146 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:38292 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1184 (was 1114) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=772 (was 700) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2600 (was 3606) 2024-12-10T19:28:55,159 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-12-10T19:28:55,171 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=425, OpenFileDescriptor=1184, MaxFileDescriptor=1048576, SystemLoadAverage=772, ProcessCount=11, AvailableMemoryMB=2600 2024-12-10T19:28:55,171 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-12-10T19:28:55,186 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:55,189 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:55,190 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:55,192 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-22250553, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-22250553, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:55,210 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-22250553/hregion-22250553.1733858935192, exclude list is [], retry=0 2024-12-10T19:28:55,213 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:55,213 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:55,213 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:55,247 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-22250553/hregion-22250553.1733858935192 2024-12-10T19:28:55,248 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:55,248 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ecf4c9d5cf03e7c7c103e5e95538a77e, NAME => 'testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:28:55,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741910_1088 (size=61) 2024-12-10T19:28:55,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741910_1088 (size=61) 2024-12-10T19:28:55,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741910_1088 (size=61) 2024-12-10T19:28:55,259 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:55,261 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,262 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecf4c9d5cf03e7c7c103e5e95538a77e columnFamilyName a 2024-12-10T19:28:55,263 DEBUG [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:55,263 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] regionserver.HStore(327): Store=ecf4c9d5cf03e7c7c103e5e95538a77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:55,263 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,264 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,264 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,266 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,268 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:55,268 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ecf4c9d5cf03e7c7c103e5e95538a77e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60970019, jitterRate=-0.09147591888904572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T19:28:55,269 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ecf4c9d5cf03e7c7c103e5e95538a77e: Writing region info on filesystem at 1733858935259Initializing all the Stores at 1733858935260 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858935260Cleaning up temporary data from old regions at 1733858935264 (+4 ms)Region opened successfully at 1733858935269 (+5 ms) 2024-12-10T19:28:55,269 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ecf4c9d5cf03e7c7c103e5e95538a77e, disabling compactions & flushes 2024-12-10T19:28:55,269 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. 2024-12-10T19:28:55,269 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. 2024-12-10T19:28:55,269 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. after waiting 0 ms 2024-12-10T19:28:55,269 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. 2024-12-10T19:28:55,269 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. 2024-12-10T19:28:55,269 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ecf4c9d5cf03e7c7c103e5e95538a77e: Waiting for close lock at 1733858935269Disabling compacts and flushes for region at 1733858935269Disabling writes for close at 1733858935269Writing region close event to WAL at 1733858935269Closed at 1733858935269 2024-12-10T19:28:55,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741909_1087 (size=175) 2024-12-10T19:28:55,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741909_1087 (size=175) 2024-12-10T19:28:55,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741909_1087 (size=175) 2024-12-10T19:28:55,275 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:55,275 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-22250553:(num 1733858935192) 2024-12-10T19:28:55,275 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:55,276 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:55,291 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277, exclude list is [], retry=0 2024-12-10T19:28:55,294 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:55,295 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:55,295 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:55,335 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277 2024-12-10T19:28:55,336 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:55,336 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ecf4c9d5cf03e7c7c103e5e95538a77e, NAME => 'testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:55,337 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:55,337 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,337 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,348 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,350 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecf4c9d5cf03e7c7c103e5e95538a77e columnFamilyName a 2024-12-10T19:28:55,350 DEBUG [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:55,351 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] regionserver.HStore(327): Store=ecf4c9d5cf03e7c7c103e5e95538a77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:55,351 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,352 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,354 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,355 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,355 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,359 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:55,360 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ecf4c9d5cf03e7c7c103e5e95538a77e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62876757, jitterRate=-0.06306330859661102}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T19:28:55,361 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ecf4c9d5cf03e7c7c103e5e95538a77e: Writing region info on filesystem at 1733858935337Initializing all the Stores at 1733858935338 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858935338Cleaning up temporary data from old regions at 1733858935355 (+17 ms)Region opened successfully at 1733858935361 (+6 ms) 2024-12-10T19:28:55,380 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ecf4c9d5cf03e7c7c103e5e95538a77e, disabling compactions & flushes 2024-12-10T19:28:55,380 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. 2024-12-10T19:28:55,380 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. 2024-12-10T19:28:55,380 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. after waiting 0 ms 2024-12-10T19:28:55,380 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. 2024-12-10T19:28:55,380 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. 2024-12-10T19:28:55,381 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. 2024-12-10T19:28:55,381 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ecf4c9d5cf03e7c7c103e5e95538a77e: Waiting for close lock at 1733858935380Disabling compacts and flushes for region at 1733858935380Disabling writes for close at 1733858935380Writing region close event to WAL at 1733858935380Closed at 1733858935381 (+1 ms) 2024-12-10T19:28:55,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741911_1089 (size=1795) 2024-12-10T19:28:55,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741911_1089 (size=1795) 2024-12-10T19:28:55,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741911_1089 (size=1795) 2024-12-10T19:28:55,427 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277, size=1.8 K (1795bytes) 2024-12-10T19:28:55,427 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277 2024-12-10T19:28:55,428 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277 after 0ms 2024-12-10T19:28:55,462 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:55,462 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277 took 36ms 2024-12-10T19:28:55,464 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277 so closing down 2024-12-10T19:28:55,464 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:28:55,466 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733858935277.temp 2024-12-10T19:28:55,500 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000003-wal.1733858935277.temp 2024-12-10T19:28:55,501 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:28:55,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741912_1090 (size=1795) 2024-12-10T19:28:55,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741912_1090 (size=1795) 2024-12-10T19:28:55,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741912_1090 (size=1795) 2024-12-10T19:28:55,522 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000003-wal.1733858935277.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-10T19:28:55,529 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000003-wal.1733858935277.temp to hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000012 2024-12-10T19:28:55,529 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-10T19:28:55,529 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277, journal: Splitting hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277, size=1.8 K (1795bytes) at 1733858935427Finishing writing output for hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277 so closing down at 1733858935464 (+37 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000003-wal.1733858935277.temp at 1733858935501 (+37 ms)3 split writer threads finished at 1733858935501Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000003-wal.1733858935277.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733858935522 (+21 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000003-wal.1733858935277.temp to hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000012 at 1733858935529 (+7 ms)Processed 10 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733858935529 2024-12-10T19:28:55,534 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858935277 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858935277 2024-12-10T19:28:55,538 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000012 2024-12-10T19:28:55,587 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:56,374 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:56,377 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:56,391 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858936377, exclude list is [], retry=0 2024-12-10T19:28:56,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:56,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:56,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:56,441 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858936377 2024-12-10T19:28:56,444 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:28:56,444 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ecf4c9d5cf03e7c7c103e5e95538a77e, NAME => 'testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:56,445 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:56,445 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,445 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,451 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,456 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecf4c9d5cf03e7c7c103e5e95538a77e columnFamilyName a 2024-12-10T19:28:56,456 DEBUG [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:56,457 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] regionserver.HStore(327): Store=ecf4c9d5cf03e7c7c103e5e95538a77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:56,457 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,458 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,460 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,461 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000012 2024-12-10T19:28:56,500 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:28:56,501 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000012 2024-12-10T19:28:56,501 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ecf4c9d5cf03e7c7c103e5e95538a77e 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-10T19:28:56,533 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/.tmp/a/bd1299a2ef8d4ef385015b4589e23f4c is 79, key is testDatalossWhenInputError/a:x0/1733858935361/Put/seqid=0 2024-12-10T19:28:56,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741914_1092 (size=5808) 2024-12-10T19:28:56,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741914_1092 (size=5808) 2024-12-10T19:28:56,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741914_1092 (size=5808) 2024-12-10T19:28:56,584 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/.tmp/a/bd1299a2ef8d4ef385015b4589e23f4c 2024-12-10T19:28:56,599 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/.tmp/a/bd1299a2ef8d4ef385015b4589e23f4c as hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/a/bd1299a2ef8d4ef385015b4589e23f4c 2024-12-10T19:28:56,620 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/a/bd1299a2ef8d4ef385015b4589e23f4c, entries=10, sequenceid=12, filesize=5.7 K 2024-12-10T19:28:56,620 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ecf4c9d5cf03e7c7c103e5e95538a77e in 119ms, sequenceid=12, compaction requested=false; wal=null 2024-12-10T19:28:56,622 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/0000000000000000012 2024-12-10T19:28:56,624 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,624 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,630 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,636 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-10T19:28:56,637 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ecf4c9d5cf03e7c7c103e5e95538a77e; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63080701, jitterRate=-0.06002430617809296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T19:28:56,638 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ecf4c9d5cf03e7c7c103e5e95538a77e: Writing region info on filesystem at 1733858936445Initializing all the Stores at 1733858936449 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858936450 (+1 ms)Obtaining lock to block concurrent updates at 1733858936501 (+51 ms)Preparing flush snapshotting stores in ecf4c9d5cf03e7c7c103e5e95538a77e at 1733858936501Finished memstore snapshotting testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733858936502 (+1 ms)Flushing stores of testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e. at 1733858936502Flushing ecf4c9d5cf03e7c7c103e5e95538a77e/a: creating writer at 1733858936502Flushing ecf4c9d5cf03e7c7c103e5e95538a77e/a: appending metadata at 1733858936532 (+30 ms)Flushing ecf4c9d5cf03e7c7c103e5e95538a77e/a: closing flushed file at 1733858936532Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b17cafb: reopening flushed file at 1733858936596 (+64 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ecf4c9d5cf03e7c7c103e5e95538a77e in 119ms, sequenceid=12, compaction requested=false; wal=null at 1733858936621 (+25 ms)Cleaning up temporary data from old regions at 1733858936624 (+3 ms)Region opened successfully at 1733858936638 (+14 ms) 2024-12-10T19:28:56,643 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ecf4c9d5cf03e7c7c103e5e95538a77e, NAME => 'testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:56,643 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733858935187.ecf4c9d5cf03e7c7c103e5e95538a77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:56,643 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,643 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,648 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,652 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecf4c9d5cf03e7c7c103e5e95538a77e columnFamilyName a 2024-12-10T19:28:56,652 DEBUG [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:56,676 DEBUG [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/a/bd1299a2ef8d4ef385015b4589e23f4c 2024-12-10T19:28:56,676 INFO [StoreOpener-ecf4c9d5cf03e7c7c103e5e95538a77e-1 {}] regionserver.HStore(327): Store=ecf4c9d5cf03e7c7c103e5e95538a77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:56,676 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,680 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,683 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,684 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,684 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,686 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ecf4c9d5cf03e7c7c103e5e95538a77e 2024-12-10T19:28:56,690 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testDatalossWhenInputError/ecf4c9d5cf03e7c7c103e5e95538a77e/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-10T19:28:56,692 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ecf4c9d5cf03e7c7c103e5e95538a77e; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62150132, jitterRate=-0.07389086484909058}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T19:28:56,692 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ecf4c9d5cf03e7c7c103e5e95538a77e: Writing region info on filesystem at 1733858936644Initializing all the Stores at 1733858936646 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858936646Cleaning up temporary data from old regions at 1733858936684 (+38 ms)Region opened successfully at 1733858936692 (+8 ms) 2024-12-10T19:28:56,714 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=432 (was 425) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:49126 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:49220 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34896 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:38316 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:34968 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:38292 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1608359495-172.17.0.3-1733858888650:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1184) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=863 (was 772) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2460 (was 2600) 2024-12-10T19:28:56,715 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-10T19:28:56,730 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=432, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=863, ProcessCount=11, AvailableMemoryMB=2458 2024-12-10T19:28:56,730 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-10T19:28:56,752 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:56,754 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:28:56,755 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:28:56,758 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-02858762, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-02858762, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:56,778 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-02858762/hregion-02858762.1733858936759, exclude list is [], retry=0 2024-12-10T19:28:56,781 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:56,782 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:56,784 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:56,833 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-02858762/hregion-02858762.1733858936759 2024-12-10T19:28:56,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:56,849 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5f90e5550788f26fb015c3b9a0f9ec0a, NAME => 'testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:28:56,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741916_1094 (size=63) 2024-12-10T19:28:56,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741916_1094 (size=63) 2024-12-10T19:28:56,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741916_1094 (size=63) 2024-12-10T19:28:56,874 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:56,876 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,889 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f90e5550788f26fb015c3b9a0f9ec0a columnFamilyName a 2024-12-10T19:28:56,889 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:56,890 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(327): Store=5f90e5550788f26fb015c3b9a0f9ec0a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:56,890 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,892 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f90e5550788f26fb015c3b9a0f9ec0a columnFamilyName b 2024-12-10T19:28:56,892 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:56,895 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(327): Store=5f90e5550788f26fb015c3b9a0f9ec0a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:56,896 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,898 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f90e5550788f26fb015c3b9a0f9ec0a columnFamilyName c 2024-12-10T19:28:56,898 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:56,900 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(327): Store=5f90e5550788f26fb015c3b9a0f9ec0a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:56,900 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,901 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,901 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,903 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:56,906 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,909 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:28:56,910 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5f90e5550788f26fb015c3b9a0f9ec0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62752748, jitterRate=-0.06491118669509888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:56,910 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5f90e5550788f26fb015c3b9a0f9ec0a: Writing region info on filesystem at 1733858936874Initializing all the Stores at 1733858936875 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858936876 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858936876Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858936876Cleaning up temporary data from old regions at 1733858936903 (+27 ms)Region opened successfully at 1733858936910 (+7 ms) 2024-12-10T19:28:56,910 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5f90e5550788f26fb015c3b9a0f9ec0a, disabling compactions & flushes 2024-12-10T19:28:56,910 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. 2024-12-10T19:28:56,910 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. 2024-12-10T19:28:56,911 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. after waiting 0 ms 2024-12-10T19:28:56,911 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. 2024-12-10T19:28:56,911 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. 2024-12-10T19:28:56,911 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5f90e5550788f26fb015c3b9a0f9ec0a: Waiting for close lock at 1733858936910Disabling compacts and flushes for region at 1733858936910Disabling writes for close at 1733858936911 (+1 ms)Writing region close event to WAL at 1733858936911Closed at 1733858936911 2024-12-10T19:28:56,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741915_1093 (size=175) 2024-12-10T19:28:56,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741915_1093 (size=175) 2024-12-10T19:28:56,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741915_1093 (size=175) 2024-12-10T19:28:56,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:28:56,921 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-02858762:(num 1733858936759) 2024-12-10T19:28:56,921 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:28:56,924 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:28:56,943 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925, exclude list is [], retry=0 2024-12-10T19:28:56,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:28:56,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:28:56,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:28:56,979 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 2024-12-10T19:28:56,979 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:28:56,979 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f90e5550788f26fb015c3b9a0f9ec0a, NAME => 'testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:28:56,980 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:28:56,980 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,980 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,984 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,987 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f90e5550788f26fb015c3b9a0f9ec0a columnFamilyName a 2024-12-10T19:28:56,988 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:56,988 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(327): Store=5f90e5550788f26fb015c3b9a0f9ec0a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:56,989 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,992 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f90e5550788f26fb015c3b9a0f9ec0a columnFamilyName b 2024-12-10T19:28:56,992 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:56,993 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(327): Store=5f90e5550788f26fb015c3b9a0f9ec0a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:56,993 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,995 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f90e5550788f26fb015c3b9a0f9ec0a columnFamilyName c 2024-12-10T19:28:56,996 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:28:56,997 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(327): Store=5f90e5550788f26fb015c3b9a0f9ec0a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:28:56,997 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,998 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:56,999 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:57,001 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:57,001 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:57,002 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:28:57,003 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:28:57,004 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5f90e5550788f26fb015c3b9a0f9ec0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63293243, jitterRate=-0.05685718357563019}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:28:57,005 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5f90e5550788f26fb015c3b9a0f9ec0a: Writing region info on filesystem at 1733858936980Initializing all the Stores at 1733858936983 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858936983Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858936984 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858936984Cleaning up temporary data from old regions at 1733858937001 (+17 ms)Region opened successfully at 1733858937005 (+4 ms) 2024-12-10T19:28:57,016 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733858937016/Put/seqid=0 2024-12-10T19:28:57,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741918_1096 (size=4875) 2024-12-10T19:28:57,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741918_1096 (size=4875) 2024-12-10T19:28:57,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741918_1096 (size=4875) 2024-12-10T19:28:57,034 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733858937034/Put/seqid=0 2024-12-10T19:28:57,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741919_1097 (size=4875) 2024-12-10T19:28:57,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741919_1097 (size=4875) 2024-12-10T19:28:57,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741919_1097 (size=4875) 2024-12-10T19:28:57,059 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733858937058/Put/seqid=0 2024-12-10T19:28:57,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741920_1098 (size=4875) 2024-12-10T19:28:57,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741920_1098 (size=4875) 2024-12-10T19:28:57,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741920_1098 (size=4875) 2024-12-10T19:28:57,077 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 5f90e5550788f26fb015c3b9a0f9ec0a/a 2024-12-10T19:28:57,084 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-10T19:28:57,084 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T19:28:57,085 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 5f90e5550788f26fb015c3b9a0f9ec0a/a 2024-12-10T19:28:57,093 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-10T19:28:57,093 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T19:28:57,093 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 5f90e5550788f26fb015c3b9a0f9ec0a/a 2024-12-10T19:28:57,098 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-10T19:28:57,098 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T19:28:57,099 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5f90e5550788f26fb015c3b9a0f9ec0a 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-10T19:28:57,116 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/.tmp/a/74b27940f90e4afa98678d1861a4f15c is 55, key is testCompactedBulkLoadedFiles/a:a/1733858937006/Put/seqid=0 2024-12-10T19:28:57,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741921_1099 (size=5107) 2024-12-10T19:28:57,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741921_1099 (size=5107) 2024-12-10T19:28:57,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741921_1099 (size=5107) 2024-12-10T19:28:57,139 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/.tmp/a/74b27940f90e4afa98678d1861a4f15c 2024-12-10T19:28:57,148 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/.tmp/a/74b27940f90e4afa98678d1861a4f15c as hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/74b27940f90e4afa98678d1861a4f15c 2024-12-10T19:28:57,154 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/74b27940f90e4afa98678d1861a4f15c, entries=1, sequenceid=4, filesize=5.0 K 2024-12-10T19:28:57,155 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 5f90e5550788f26fb015c3b9a0f9ec0a in 57ms, sequenceid=4, compaction requested=false 2024-12-10T19:28:57,155 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5f90e5550788f26fb015c3b9a0f9ec0a: 2024-12-10T19:28:57,158 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_ 2024-12-10T19:28:57,159 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_ 2024-12-10T19:28:57,160 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_ 2024-12-10T19:28:57,161 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile0 into 5f90e5550788f26fb015c3b9a0f9ec0a/a as hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_ - updating store file list. 2024-12-10T19:28:57,168 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 668d112b643d4a27b9f5c684a961a7f3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T19:28:57,168 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_ into 5f90e5550788f26fb015c3b9a0f9ec0a/a 2024-12-10T19:28:57,168 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile0 into 5f90e5550788f26fb015c3b9a0f9ec0a/a (new location: hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_) 2024-12-10T19:28:57,170 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile1 into 5f90e5550788f26fb015c3b9a0f9ec0a/a as hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_ - updating store file list. 2024-12-10T19:28:57,176 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T19:28:57,176 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_ into 5f90e5550788f26fb015c3b9a0f9ec0a/a 2024-12-10T19:28:57,176 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile1 into 5f90e5550788f26fb015c3b9a0f9ec0a/a (new location: hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_) 2024-12-10T19:28:57,178 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile2 into 5f90e5550788f26fb015c3b9a0f9ec0a/a as hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_ - updating store file list. 2024-12-10T19:28:57,185 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T19:28:57,185 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_ into 5f90e5550788f26fb015c3b9a0f9ec0a/a 2024-12-10T19:28:57,185 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46813/hbase/testCompactedBulkLoadedFiles/hfile2 into 5f90e5550788f26fb015c3b9a0f9ec0a/a (new location: hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_) 2024-12-10T19:28:57,204 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T19:28:57,204 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5f90e5550788f26fb015c3b9a0f9ec0a/a is initiating major compaction (all files) 2024-12-10T19:28:57,204 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5f90e5550788f26fb015c3b9a0f9ec0a/a in testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. 2024-12-10T19:28:57,204 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/74b27940f90e4afa98678d1861a4f15c, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_] into tmpdir=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/.tmp, totalSize=19.3 K 2024-12-10T19:28:57,205 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 74b27940f90e4afa98678d1861a4f15c, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733858937006 2024-12-10T19:28:57,206 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 668d112b643d4a27b9f5c684a961a7f3_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-10T19:28:57,206 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-10T19:28:57,206 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-10T19:28:57,231 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/.tmp/a/0985dbd5314d4479b7caaa85f0bd811b is 55, key is testCompactedBulkLoadedFiles/a:a/1733858937006/Put/seqid=0 2024-12-10T19:28:57,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741922_1100 (size=6154) 2024-12-10T19:28:57,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741922_1100 (size=6154) 2024-12-10T19:28:57,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741922_1100 (size=6154) 2024-12-10T19:28:57,251 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/.tmp/a/0985dbd5314d4479b7caaa85f0bd811b as hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/0985dbd5314d4479b7caaa85f0bd811b 2024-12-10T19:28:57,262 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 5f90e5550788f26fb015c3b9a0f9ec0a/a of 5f90e5550788f26fb015c3b9a0f9ec0a into 0985dbd5314d4479b7caaa85f0bd811b(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T19:28:57,262 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5f90e5550788f26fb015c3b9a0f9ec0a: 2024-12-10T19:28:57,262 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-10T19:28:57,262 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-10T19:28:57,318 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925, size=0 (0bytes) 2024-12-10T19:28:57,318 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 might be still open, length is 0 2024-12-10T19:28:57,318 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 2024-12-10T19:28:57,319 WARN [IPC Server handler 3 on default port 46813 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-10T19:28:57,320 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 after 2ms 2024-12-10T19:28:57,449 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:35010 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:43797:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35010 dst: /127.0.0.1:43797 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43797 remote=/127.0.0.1:35010]. Total timeout mills is 60000, 59812 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:57,449 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:38344 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39621:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38344 dst: /127.0.0.1:39621 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:57,450 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:49268 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:35009:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49268 dst: /127.0.0.1:35009 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:28:57,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741917_1101 (size=1794) 2024-12-10T19:28:58,243 INFO [master/e4ad70981cbb:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-10T19:28:58,243 INFO [master/e4ad70981cbb:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-10T19:29:00,156 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T19:29:01,325 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 after 4007ms 2024-12-10T19:29:01,367 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:29:01,368 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 took 4050ms 2024-12-10T19:29:01,369 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925; continuing. 2024-12-10T19:29:01,369 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 so closing down 2024-12-10T19:29:01,369 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:29:01,371 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733858936925.temp 2024-12-10T19:29:01,406 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000003-wal.1733858936925.temp 2024-12-10T19:29:01,407 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:29:01,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741923_1102 (size=800) 2024-12-10T19:29:01,413 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000003-wal.1733858936925.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-10T19:29:01,415 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000003-wal.1733858936925.temp to hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000008 2024-12-10T19:29:01,415 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 47 ms; skipped=3; WAL=hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T19:29:01,415 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925, journal: Splitting hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925, size=0 (0bytes) at 1733858937318Finishing writing output for hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 so closing down at 1733858941369 (+4051 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000003-wal.1733858936925.temp at 1733858941406 (+37 ms)3 split writer threads finished at 1733858941407 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000003-wal.1733858936925.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733858941413 (+6 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000003-wal.1733858936925.temp to hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000008 at 1733858941415 (+2 ms)Processed 5 edits across 1 Regions in 47 ms; skipped=3; WAL=hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925, size=0, length=0, corrupted=false, cancelled=false at 1733858941415 2024-12-10T19:29:01,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741923_1102 (size=800) 2024-12-10T19:29:01,417 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858936925 2024-12-10T19:29:01,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741923_1102 (size=800) 2024-12-10T19:29:01,418 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000008 2024-12-10T19:29:01,418 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:29:01,424 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:29:01,438 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858941424, exclude list is [], retry=0 2024-12-10T19:29:01,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:29:01,441 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:29:01,441 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:29:01,470 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858941424 2024-12-10T19:29:01,471 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539)] 2024-12-10T19:29:01,471 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f90e5550788f26fb015c3b9a0f9ec0a, NAME => 'testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:29:01,471 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:29:01,471 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,471 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,472 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,473 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f90e5550788f26fb015c3b9a0f9ec0a columnFamilyName a 2024-12-10T19:29:01,473 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:01,481 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/0985dbd5314d4479b7caaa85f0bd811b 2024-12-10T19:29:01,485 DEBUG [StoreFileOpener-5f90e5550788f26fb015c3b9a0f9ec0a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 668d112b643d4a27b9f5c684a961a7f3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T19:29:01,486 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_ 2024-12-10T19:29:01,494 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/74b27940f90e4afa98678d1861a4f15c 2024-12-10T19:29:01,500 DEBUG [StoreFileOpener-5f90e5550788f26fb015c3b9a0f9ec0a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T19:29:01,500 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_ 2024-12-10T19:29:01,505 DEBUG [StoreFileOpener-5f90e5550788f26fb015c3b9a0f9ec0a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T19:29:01,505 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_ 2024-12-10T19:29:01,505 WARN [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e25d172 2024-12-10T19:29:01,505 WARN [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/74b27940f90e4afa98678d1861a4f15c from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e25d172 2024-12-10T19:29:01,505 WARN [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e25d172 2024-12-10T19:29:01,506 WARN [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e25d172 2024-12-10T19:29:01,506 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/74b27940f90e4afa98678d1861a4f15c, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_] to archive 2024-12-10T19:29:01,507 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T19:29:01,509 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_ to hdfs://localhost:46813/hbase/archive/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/668d112b643d4a27b9f5c684a961a7f3_SeqId_4_ 2024-12-10T19:29:01,509 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/74b27940f90e4afa98678d1861a4f15c to hdfs://localhost:46813/hbase/archive/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/74b27940f90e4afa98678d1861a4f15c 2024-12-10T19:29:01,510 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_ to hdfs://localhost:46813/hbase/archive/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_ 2024-12-10T19:29:01,510 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_ to hdfs://localhost:46813/hbase/archive/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/a/97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_ 2024-12-10T19:29:01,510 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(327): Store=5f90e5550788f26fb015c3b9a0f9ec0a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:01,511 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,511 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f90e5550788f26fb015c3b9a0f9ec0a columnFamilyName b 2024-12-10T19:29:01,511 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:01,512 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(327): Store=5f90e5550788f26fb015c3b9a0f9ec0a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:01,512 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,513 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f90e5550788f26fb015c3b9a0f9ec0a columnFamilyName c 2024-12-10T19:29:01,513 DEBUG [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:01,513 INFO [StoreOpener-5f90e5550788f26fb015c3b9a0f9ec0a-1 {}] regionserver.HStore(327): Store=5f90e5550788f26fb015c3b9a0f9ec0a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:01,514 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,515 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,516 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,517 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000008 2024-12-10T19:29:01,556 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:29:01,558 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 5f90e5550788f26fb015c3b9a0f9ec0a : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "5f90e5550788f26fb015c3b9a0f9ec0a" family_name: "a" compaction_input: "74b27940f90e4afa98678d1861a4f15c" compaction_input: "668d112b643d4a27b9f5c684a961a7f3_SeqId_4_" compaction_input: "85edc1ed339c4cdd99f378466d5cfd5e_SeqId_4_" compaction_input: "97dd6cfe325b40cabb2fe47c1c2dd662_SeqId_4_" compaction_output: "0985dbd5314d4479b7caaa85f0bd811b" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-10T19:29:01,558 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-10T19:29:01,559 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000008 2024-12-10T19:29:01,560 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/0000000000000000008 2024-12-10T19:29:01,561 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,561 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,562 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:29:01,564 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5f90e5550788f26fb015c3b9a0f9ec0a 2024-12-10T19:29:01,568 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testCompactedBulkLoadedFiles/5f90e5550788f26fb015c3b9a0f9ec0a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-10T19:29:01,570 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5f90e5550788f26fb015c3b9a0f9ec0a; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59284546, jitterRate=-0.1165914237499237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:29:01,570 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5f90e5550788f26fb015c3b9a0f9ec0a: Writing region info on filesystem at 1733858941471Initializing all the Stores at 1733858941472 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858941472Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858941472Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858941472Cleaning up temporary data from old regions at 1733858941561 (+89 ms)Region opened successfully at 1733858941570 (+9 ms) 2024-12-10T19:29:01,573 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5f90e5550788f26fb015c3b9a0f9ec0a, disabling compactions & flushes 2024-12-10T19:29:01,573 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. 2024-12-10T19:29:01,573 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. 2024-12-10T19:29:01,573 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. after waiting 0 ms 2024-12-10T19:29:01,573 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. 2024-12-10T19:29:01,577 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733858936752.5f90e5550788f26fb015c3b9a0f9ec0a. 2024-12-10T19:29:01,577 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5f90e5550788f26fb015c3b9a0f9ec0a: Waiting for close lock at 1733858941573Disabling compacts and flushes for region at 1733858941573Disabling writes for close at 1733858941573Writing region close event to WAL at 1733858941577 (+4 ms)Closed at 1733858941577 2024-12-10T19:29:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741924_1103 (size=175) 2024-12-10T19:29:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741924_1103 (size=175) 2024-12-10T19:29:01,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741924_1103 (size=175) 2024-12-10T19:29:01,598 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:29:01,598 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733858941424) 2024-12-10T19:29:01,632 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=445 (was 432) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2142663679_22 at /127.0.0.1:54824 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:46813 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:46813 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2142663679_22 at /127.0.0.1:60532 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1341 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=841 (was 863), ProcessCount=11 (was 11), AvailableMemoryMB=2501 (was 2458) - AvailableMemoryMB LEAK? - 2024-12-10T19:29:01,632 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1341 is superior to 1024 2024-12-10T19:29:01,647 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445, OpenFileDescriptor=1341, MaxFileDescriptor=1048576, SystemLoadAverage=841, ProcessCount=11, AvailableMemoryMB=2499 2024-12-10T19:29:01,647 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1341 is superior to 1024 2024-12-10T19:29:01,667 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:29:01,670 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T19:29:01,671 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T19:29:01,674 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65611698, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/hregion-65611698, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:29:01,688 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65611698/hregion-65611698.1733858941674, exclude list is [], retry=0 2024-12-10T19:29:01,694 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:29:01,695 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:29:01,695 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:29:01,745 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65611698/hregion-65611698.1733858941674 2024-12-10T19:29:01,746 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:45539:45539)] 2024-12-10T19:29:01,746 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c563fd3f8abcded7707f15b4a94dce7a, NAME => 'testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46813/hbase 2024-12-10T19:29:01,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741926_1105 (size=67) 2024-12-10T19:29:01,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741926_1105 (size=67) 2024-12-10T19:29:01,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741926_1105 (size=67) 2024-12-10T19:29:01,759 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:29:01,761 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,763 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName a 2024-12-10T19:29:01,763 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:01,764 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:01,764 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,769 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName b 2024-12-10T19:29:01,770 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:01,771 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:01,771 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,773 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName c 2024-12-10T19:29:01,773 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:01,774 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:01,779 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,783 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,783 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,786 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,786 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,786 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:29:01,792 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,801 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T19:29:01,801 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c563fd3f8abcded7707f15b4a94dce7a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60657543, jitterRate=-0.09613217413425446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:29:01,801 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c563fd3f8abcded7707f15b4a94dce7a: Writing region info on filesystem at 1733858941759Initializing all the Stores at 1733858941760 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858941760Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858941761 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858941761Cleaning up temporary data from old regions at 1733858941786 (+25 ms)Region opened successfully at 1733858941801 (+15 ms) 2024-12-10T19:29:01,802 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c563fd3f8abcded7707f15b4a94dce7a, disabling compactions & flushes 2024-12-10T19:29:01,802 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:01,802 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:01,802 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. after waiting 0 ms 2024-12-10T19:29:01,802 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:01,804 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:01,804 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c563fd3f8abcded7707f15b4a94dce7a: Waiting for close lock at 1733858941802Disabling compacts and flushes for region at 1733858941802Disabling writes for close at 1733858941802Writing region close event to WAL at 1733858941804 (+2 ms)Closed at 1733858941804 2024-12-10T19:29:01,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741925_1104 (size=175) 2024-12-10T19:29:01,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741925_1104 (size=175) 2024-12-10T19:29:01,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741925_1104 (size=175) 2024-12-10T19:29:01,818 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:29:01,818 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65611698:(num 1733858941674) 2024-12-10T19:29:01,818 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:29:01,821 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:29:01,837 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821, exclude list is [], retry=0 2024-12-10T19:29:01,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:29:01,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:29:01,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:29:01,895 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821 2024-12-10T19:29:01,896 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811),(127.0.0.1/127.0.0.1:44649:44649)] 2024-12-10T19:29:01,896 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c563fd3f8abcded7707f15b4a94dce7a, NAME => 'testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:29:01,896 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:29:01,896 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,896 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,902 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,904 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName a 2024-12-10T19:29:01,904 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:01,905 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:01,905 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,909 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName b 2024-12-10T19:29:01,909 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:01,910 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:01,910 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,912 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName c 2024-12-10T19:29:01,913 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:01,913 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:01,914 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,915 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,918 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,919 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,919 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,920 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:29:01,924 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:01,926 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c563fd3f8abcded7707f15b4a94dce7a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58883335, jitterRate=-0.12256993353366852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:29:01,926 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c563fd3f8abcded7707f15b4a94dce7a: Writing region info on filesystem at 1733858941896Initializing all the Stores at 1733858941901 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858941901Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858941901Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858941901Cleaning up temporary data from old regions at 1733858941919 (+18 ms)Region opened successfully at 1733858941926 (+7 ms) 2024-12-10T19:29:01,981 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c563fd3f8abcded7707f15b4a94dce7a 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-10T19:29:01,999 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/a/04bd550408964161b18518a8f6271170 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733858941926/Put/seqid=0 2024-12-10T19:29:02,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741928_1107 (size=5958) 2024-12-10T19:29:02,052 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/a/04bd550408964161b18518a8f6271170 2024-12-10T19:29:02,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741928_1107 (size=5958) 2024-12-10T19:29:02,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741928_1107 (size=5958) 2024-12-10T19:29:02,072 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/a/04bd550408964161b18518a8f6271170 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/a/04bd550408964161b18518a8f6271170 2024-12-10T19:29:02,098 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/a/04bd550408964161b18518a8f6271170, entries=10, sequenceid=13, filesize=5.8 K 2024-12-10T19:29:02,100 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c563fd3f8abcded7707f15b4a94dce7a in 118ms, sequenceid=13, compaction requested=false 2024-12-10T19:29:02,100 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c563fd3f8abcded7707f15b4a94dce7a: 2024-12-10T19:29:02,173 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c563fd3f8abcded7707f15b4a94dce7a, disabling compactions & flushes 2024-12-10T19:29:02,173 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:02,173 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:02,173 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. after waiting 0 ms 2024-12-10T19:29:02,173 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:02,192 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:02,192 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:02,193 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c563fd3f8abcded7707f15b4a94dce7a: Waiting for close lock at 1733858942173Disabling compacts and flushes for region at 1733858942173Disabling writes for close at 1733858942173Writing region close event to WAL at 1733858942192 (+19 ms)Closed at 1733858942192 2024-12-10T19:29:02,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741927_1106 (size=6243) 2024-12-10T19:29:02,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741927_1106 (size=6243) 2024-12-10T19:29:02,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741927_1106 (size=6243) 2024-12-10T19:29:02,216 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821 not finished, retry = 0 2024-12-10T19:29:02,352 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821, size=6.1 K (6243bytes) 2024-12-10T19:29:02,352 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821 2024-12-10T19:29:02,353 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821 after 1ms 2024-12-10T19:29:02,404 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:29:02,405 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821 took 53ms 2024-12-10T19:29:02,417 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821 so closing down 2024-12-10T19:29:02,417 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:29:02,419 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733858941821.temp 2024-12-10T19:29:02,461 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000003-wal.1733858941821.temp 2024-12-10T19:29:02,463 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:29:02,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741929_1108 (size=5575) 2024-12-10T19:29:02,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741929_1108 (size=5575) 2024-12-10T19:29:02,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741929_1108 (size=5575) 2024-12-10T19:29:02,907 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000003-wal.1733858941821.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-10T19:29:02,909 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000003-wal.1733858941821.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000035 2024-12-10T19:29:02,909 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 499 ms; skipped=2; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-10T19:29:02,909 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821, journal: Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821, size=6.1 K (6243bytes) at 1733858942352Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821 so closing down at 1733858942417 (+65 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000003-wal.1733858941821.temp at 1733858942461 (+44 ms)3 split writer threads finished at 1733858942463 (+2 ms)Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000003-wal.1733858941821.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733858942907 (+444 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000003-wal.1733858941821.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000035 at 1733858942909 (+2 ms)Processed 32 edits across 1 Regions in 499 ms; skipped=2; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733858942909 2024-12-10T19:29:02,911 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858941821 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858941821 2024-12-10T19:29:02,913 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000035 2024-12-10T19:29:02,913 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:29:02,915 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:29:02,934 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916, exclude list is [], retry=0 2024-12-10T19:29:02,937 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:29:02,938 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:29:02,938 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:29:02,986 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 2024-12-10T19:29:02,987 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:29:02,987 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c563fd3f8abcded7707f15b4a94dce7a, NAME => 'testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T19:29:02,987 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:29:02,987 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:02,987 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:02,996 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:02,998 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName a 2024-12-10T19:29:02,998 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:03,018 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/a/04bd550408964161b18518a8f6271170 2024-12-10T19:29:03,018 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:03,018 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:03,019 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName b 2024-12-10T19:29:03,019 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:03,022 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:03,022 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:03,023 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName c 2024-12-10T19:29:03,023 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:03,024 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:03,024 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:03,026 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:03,033 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:03,034 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000035 2024-12-10T19:29:03,078 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:29:03,097 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000035 2024-12-10T19:29:03,097 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c563fd3f8abcded7707f15b4a94dce7a 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-10T19:29:03,124 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/b/9483ece5037b4f919540a40f2694ea02 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733858942100/Put/seqid=0 2024-12-10T19:29:03,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741931_1110 (size=5958) 2024-12-10T19:29:03,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741931_1110 (size=5958) 2024-12-10T19:29:03,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741931_1110 (size=5958) 2024-12-10T19:29:03,188 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/b/9483ece5037b4f919540a40f2694ea02 2024-12-10T19:29:03,299 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/c/fb0565b568e34c599e5518685984da62 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733858942151/Put/seqid=0 2024-12-10T19:29:03,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741932_1111 (size=5958) 2024-12-10T19:29:03,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741932_1111 (size=5958) 2024-12-10T19:29:03,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741932_1111 (size=5958) 2024-12-10T19:29:03,340 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/c/fb0565b568e34c599e5518685984da62 2024-12-10T19:29:03,373 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/b/9483ece5037b4f919540a40f2694ea02 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/b/9483ece5037b4f919540a40f2694ea02 2024-12-10T19:29:03,386 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/b/9483ece5037b4f919540a40f2694ea02, entries=10, sequenceid=35, filesize=5.8 K 2024-12-10T19:29:03,392 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/c/fb0565b568e34c599e5518685984da62 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/c/fb0565b568e34c599e5518685984da62 2024-12-10T19:29:03,407 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/c/fb0565b568e34c599e5518685984da62, entries=10, sequenceid=35, filesize=5.8 K 2024-12-10T19:29:03,407 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for c563fd3f8abcded7707f15b4a94dce7a in 310ms, sequenceid=35, compaction requested=false; wal=null 2024-12-10T19:29:03,408 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000035 2024-12-10T19:29:03,415 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:03,415 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:03,417 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:29:03,419 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:03,436 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-10T19:29:03,446 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-10T19:29:03,446 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-10T19:29:03,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-10T19:29:03,448 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-10T19:29:03,449 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c563fd3f8abcded7707f15b4a94dce7a; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62885606, jitterRate=-0.06293144822120667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:29:03,449 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c563fd3f8abcded7707f15b4a94dce7a: Writing region info on filesystem at 1733858942987Initializing all the Stores at 1733858942995 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858942995Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858942996 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858942996Obtaining lock to block concurrent updates at 1733858943097 (+101 ms)Preparing flush snapshotting stores in c563fd3f8abcded7707f15b4a94dce7a at 1733858943097Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733858943098 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. at 1733858943098Flushing c563fd3f8abcded7707f15b4a94dce7a/b: creating writer at 1733858943098Flushing c563fd3f8abcded7707f15b4a94dce7a/b: appending metadata at 1733858943122 (+24 ms)Flushing c563fd3f8abcded7707f15b4a94dce7a/b: closing flushed file at 1733858943122Flushing c563fd3f8abcded7707f15b4a94dce7a/c: creating writer at 1733858943238 (+116 ms)Flushing c563fd3f8abcded7707f15b4a94dce7a/c: appending metadata at 1733858943298 (+60 ms)Flushing c563fd3f8abcded7707f15b4a94dce7a/c: closing flushed file at 1733858943298Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25b613e9: reopening flushed file at 1733858943358 (+60 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cd454f7: reopening flushed file at 1733858943386 (+28 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for c563fd3f8abcded7707f15b4a94dce7a in 310ms, sequenceid=35, compaction requested=false; wal=null at 1733858943407 (+21 ms)Cleaning up temporary data from old regions at 1733858943415 (+8 ms)Region opened successfully at 1733858943449 (+34 ms) 2024-12-10T19:29:03,689 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916, size=0 (0bytes) 2024-12-10T19:29:03,689 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 might be still open, length is 0 2024-12-10T19:29:03,689 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 2024-12-10T19:29:03,690 WARN [IPC Server handler 3 on default port 46813 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-10T19:29:03,691 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 after 1ms 2024-12-10T19:29:06,447 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:36314 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39621:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36314 dst: /127.0.0.1:39621 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39621 remote=/127.0.0.1:36314]. Total timeout mills is 60000, 57182 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:29:06,449 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:54884 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:35009:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54884 dst: /127.0.0.1:35009 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35009 remote=/127.0.0.1:54884]. Total timeout mills is 60000, 57181 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:29:06,451 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1225853664_22 at /127.0.0.1:60608 [Receiving block BP-1608359495-172.17.0.3-1733858888650:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:43797:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60608 dst: /127.0.0.1:43797 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:29:06,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741930_1112 (size=5567) 2024-12-10T19:29:06,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741930_1112 (size=5567) 2024-12-10T19:29:06,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741930_1112 (size=5567) 2024-12-10T19:29:07,691 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 after 4002ms 2024-12-10T19:29:07,727 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:29:07,727 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 took 4038ms 2024-12-10T19:29:07,731 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916; continuing. 2024-12-10T19:29:07,731 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 so closing down 2024-12-10T19:29:07,731 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T19:29:07,733 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733858942916.temp 2024-12-10T19:29:07,777 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000037-wal.1733858942916.temp 2024-12-10T19:29:07,777 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T19:29:07,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741933_1113 (size=5575) 2024-12-10T19:29:07,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741933_1113 (size=5575) 2024-12-10T19:29:07,790 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000037-wal.1733858942916.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-10T19:29:07,793 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000037-wal.1733858942916.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000066 2024-12-10T19:29:07,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741933_1113 (size=5575) 2024-12-10T19:29:07,793 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T19:29:07,793 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916, journal: Splitting hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916, size=0 (0bytes) at 1733858943689Finishing writing output for hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 so closing down at 1733858947731 (+4042 ms)Creating recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000037-wal.1733858942916.temp at 1733858947777 (+46 ms)3 split writer threads finished at 1733858947777Closed recovered edits writer path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000037-wal.1733858942916.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733858947790 (+13 ms)Rename recovered edits hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000037-wal.1733858942916.temp to hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000066 at 1733858947793 (+3 ms)Processed 30 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916, size=0, length=0, corrupted=false, cancelled=false at 1733858947793 2024-12-10T19:29:07,795 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 to hdfs://localhost:46813/hbase/oldWALs/wal.1733858942916 2024-12-10T19:29:07,797 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000066 2024-12-10T19:29:07,797 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T19:29:07,801 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46813/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666, archiveDir=hdfs://localhost:46813/hbase/oldWALs, maxLogs=32 2024-12-10T19:29:07,818 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858947802, exclude list is [], retry=0 2024-12-10T19:29:07,821 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35009,DS-7224b066-4b44-401a-8843-7d969e5a27e1,DISK] 2024-12-10T19:29:07,821 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39621,DS-2a6d5c9a-5cda-464d-9274-c5378fa1f0ab,DISK] 2024-12-10T19:29:07,821 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43797,DS-4ae4a1bc-2129-46d9-a055-d8e4a384d09d,DISK] 2024-12-10T19:29:07,855 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858947802 2024-12-10T19:29:07,855 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44649:44649),(127.0.0.1/127.0.0.1:45539:45539),(127.0.0.1/127.0.0.1:41811:41811)] 2024-12-10T19:29:07,856 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T19:29:07,857 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:07,858 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName a 2024-12-10T19:29:07,858 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:07,865 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/a/04bd550408964161b18518a8f6271170 2024-12-10T19:29:07,865 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:07,865 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:07,867 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName b 2024-12-10T19:29:07,867 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:07,890 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/b/9483ece5037b4f919540a40f2694ea02 2024-12-10T19:29:07,890 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:07,890 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:07,891 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c563fd3f8abcded7707f15b4a94dce7a columnFamilyName c 2024-12-10T19:29:07,891 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T19:29:07,899 DEBUG [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/c/fb0565b568e34c599e5518685984da62 2024-12-10T19:29:07,899 INFO [StoreOpener-c563fd3f8abcded7707f15b4a94dce7a-1 {}] regionserver.HStore(327): Store=c563fd3f8abcded7707f15b4a94dce7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T19:29:07,899 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:07,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:07,902 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:07,906 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000066 2024-12-10T19:29:07,946 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-10T19:29:07,951 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000066 2024-12-10T19:29:07,951 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c563fd3f8abcded7707f15b4a94dce7a 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-10T19:29:07,968 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/a/f9d47239e1594bdf9faf8161c9062d06 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733858943472/Put/seqid=0 2024-12-10T19:29:07,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741935_1115 (size=5958) 2024-12-10T19:29:07,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741935_1115 (size=5958) 2024-12-10T19:29:07,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741935_1115 (size=5958) 2024-12-10T19:29:08,375 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/a/f9d47239e1594bdf9faf8161c9062d06 2024-12-10T19:29:08,398 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/b/6f8fb30f7a3b49f899dded41a627dc7b is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733858943548/Put/seqid=0 2024-12-10T19:29:08,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741936_1116 (size=5958) 2024-12-10T19:29:08,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741936_1116 (size=5958) 2024-12-10T19:29:08,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741936_1116 (size=5958) 2024-12-10T19:29:08,408 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/b/6f8fb30f7a3b49f899dded41a627dc7b 2024-12-10T19:29:08,428 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/c/be4d7e3453274bdb92b86b3ed6cc390d is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733858943614/Put/seqid=0 2024-12-10T19:29:08,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741937_1117 (size=5958) 2024-12-10T19:29:08,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741937_1117 (size=5958) 2024-12-10T19:29:08,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741937_1117 (size=5958) 2024-12-10T19:29:08,442 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/c/be4d7e3453274bdb92b86b3ed6cc390d 2024-12-10T19:29:08,448 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/a/f9d47239e1594bdf9faf8161c9062d06 as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/a/f9d47239e1594bdf9faf8161c9062d06 2024-12-10T19:29:08,454 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/a/f9d47239e1594bdf9faf8161c9062d06, entries=10, sequenceid=66, filesize=5.8 K 2024-12-10T19:29:08,455 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/b/6f8fb30f7a3b49f899dded41a627dc7b as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/b/6f8fb30f7a3b49f899dded41a627dc7b 2024-12-10T19:29:08,465 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/b/6f8fb30f7a3b49f899dded41a627dc7b, entries=10, sequenceid=66, filesize=5.8 K 2024-12-10T19:29:08,466 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/.tmp/c/be4d7e3453274bdb92b86b3ed6cc390d as hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/c/be4d7e3453274bdb92b86b3ed6cc390d 2024-12-10T19:29:08,472 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/c/be4d7e3453274bdb92b86b3ed6cc390d, entries=10, sequenceid=66, filesize=5.8 K 2024-12-10T19:29:08,472 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c563fd3f8abcded7707f15b4a94dce7a in 521ms, sequenceid=66, compaction requested=false; wal=null 2024-12-10T19:29:08,473 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/0000000000000000066 2024-12-10T19:29:08,475 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:08,475 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:08,476 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T19:29:08,477 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c563fd3f8abcded7707f15b4a94dce7a 2024-12-10T19:29:08,479 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/hbase/data/default/testReplayEditsWrittenViaHRegion/c563fd3f8abcded7707f15b4a94dce7a/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-10T19:29:08,481 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c563fd3f8abcded7707f15b4a94dce7a; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71939999, jitterRate=0.0719895213842392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T19:29:08,481 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c563fd3f8abcded7707f15b4a94dce7a: Writing region info on filesystem at 1733858947856Initializing all the Stores at 1733858947857 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858947857Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858947857Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733858947857Obtaining lock to block concurrent updates at 1733858947951 (+94 ms)Preparing flush snapshotting stores in c563fd3f8abcded7707f15b4a94dce7a at 1733858947951Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733858947951Flushing stores of testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. at 1733858947951Flushing c563fd3f8abcded7707f15b4a94dce7a/a: creating writer at 1733858947951Flushing c563fd3f8abcded7707f15b4a94dce7a/a: appending metadata at 1733858947968 (+17 ms)Flushing c563fd3f8abcded7707f15b4a94dce7a/a: closing flushed file at 1733858947968Flushing c563fd3f8abcded7707f15b4a94dce7a/b: creating writer at 1733858948380 (+412 ms)Flushing c563fd3f8abcded7707f15b4a94dce7a/b: appending metadata at 1733858948398 (+18 ms)Flushing c563fd3f8abcded7707f15b4a94dce7a/b: closing flushed file at 1733858948398Flushing c563fd3f8abcded7707f15b4a94dce7a/c: creating writer at 1733858948412 (+14 ms)Flushing c563fd3f8abcded7707f15b4a94dce7a/c: appending metadata at 1733858948427 (+15 ms)Flushing c563fd3f8abcded7707f15b4a94dce7a/c: closing flushed file at 1733858948427Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@534436db: reopening flushed file at 1733858948447 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ed67b6a: reopening flushed file at 1733858948454 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5577c4ed: reopening flushed file at 1733858948465 (+11 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c563fd3f8abcded7707f15b4a94dce7a in 521ms, sequenceid=66, compaction requested=false; wal=null at 1733858948473 (+8 ms)Cleaning up temporary data from old regions at 1733858948475 (+2 ms)Region opened successfully at 1733858948481 (+6 ms) 2024-12-10T19:29:08,496 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c563fd3f8abcded7707f15b4a94dce7a, disabling compactions & flushes 2024-12-10T19:29:08,496 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:08,496 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:08,496 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. after waiting 0 ms 2024-12-10T19:29:08,496 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:08,497 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733858941668.c563fd3f8abcded7707f15b4a94dce7a. 2024-12-10T19:29:08,497 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c563fd3f8abcded7707f15b4a94dce7a: Waiting for close lock at 1733858948496Disabling compacts and flushes for region at 1733858948496Disabling writes for close at 1733858948496Writing region close event to WAL at 1733858948497 (+1 ms)Closed at 1733858948497 2024-12-10T19:29:08,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741934_1114 (size=175) 2024-12-10T19:29:08,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741934_1114 (size=175) 2024-12-10T19:29:08,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741934_1114 (size=175) 2024-12-10T19:29:08,503 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T19:29:08,504 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733858947802) 2024-12-10T19:29:08,519 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=452 (was 445) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:46813 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:46813 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1874508838_22 at /127.0.0.1:36340 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1874508838_22 at /127.0.0.1:60656 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1874508838_22 at /127.0.0.1:54942 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1406 (was 1341) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=846 (was 841) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2202 (was 2499) 2024-12-10T19:29:08,519 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1406 is superior to 1024 2024-12-10T19:29:08,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T19:29:08,520 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T19:29:08,520 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T19:29:08,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:29:08,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:29:08,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T19:29:08,521 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T19:29:08,521 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=269611031, stopped=false 2024-12-10T19:29:08,522 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e4ad70981cbb,41471,1733858893036 2024-12-10T19:29:08,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T19:29:08,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T19:29:08,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T19:29:08,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:29:08,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:29:08,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:29:08,602 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T19:29:08,603 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T19:29:08,603 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T19:29:08,603 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T19:29:08,603 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T19:29:08,603 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T19:29:08,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:29:08,604 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e4ad70981cbb,39803,1733858893985' ***** 2024-12-10T19:29:08,604 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T19:29:08,604 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e4ad70981cbb,35427,1733858894073' ***** 2024-12-10T19:29:08,604 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T19:29:08,604 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T19:29:08,604 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T19:29:08,604 INFO [RS:1;e4ad70981cbb:39803 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T19:29:08,604 INFO [RS:2;e4ad70981cbb:35427 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T19:29:08,604 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T19:29:08,604 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T19:29:08,604 INFO [RS:1;e4ad70981cbb:39803 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T19:29:08,604 INFO [RS:2;e4ad70981cbb:35427 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T19:29:08,604 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(3091): Received CLOSE for 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:29:08,604 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(959): stopping server e4ad70981cbb,39803,1733858893985 2024-12-10T19:29:08,604 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T19:29:08,604 INFO [RS:1;e4ad70981cbb:39803 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e4ad70981cbb:39803. 2024-12-10T19:29:08,604 DEBUG [RS:1;e4ad70981cbb:39803 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T19:29:08,604 DEBUG [RS:1;e4ad70981cbb:39803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:29:08,605 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(976): stopping server e4ad70981cbb,39803,1733858893985; all regions closed. 2024-12-10T19:29:08,608 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(959): stopping server e4ad70981cbb,35427,1733858894073 2024-12-10T19:29:08,608 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T19:29:08,608 INFO [RS:2;e4ad70981cbb:35427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;e4ad70981cbb:35427. 2024-12-10T19:29:08,608 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 15dfd37ce6f5438f845b062fa5ab3738, disabling compactions & flushes 2024-12-10T19:29:08,608 DEBUG [RS:2;e4ad70981cbb:35427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T19:29:08,608 DEBUG [RS:2;e4ad70981cbb:35427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:29:08,608 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:29:08,608 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:29:08,608 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. after waiting 0 ms 2024-12-10T19:29:08,608 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:29:08,608 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T19:29:08,608 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T19:29:08,608 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T19:29:08,609 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T19:29:08,609 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T19:29:08,609 DEBUG [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 15dfd37ce6f5438f845b062fa5ab3738=testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738.} 2024-12-10T19:29:08,609 DEBUG [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 15dfd37ce6f5438f845b062fa5ab3738 2024-12-10T19:29:08,609 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T19:29:08,609 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T19:29:08,609 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T19:29:08,609 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T19:29:08,610 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T19:29:08,610 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-10T19:29:08,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741835_1011 (size=2764) 2024-12-10T19:29:08,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741835_1011 (size=2764) 2024-12-10T19:29:08,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741835_1011 (size=2764) 2024-12-10T19:29:08,617 DEBUG [RS:1;e4ad70981cbb:39803 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/oldWALs 2024-12-10T19:29:08,617 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/default/testReplayEditsAfterRegionMovedWithMultiCF/15dfd37ce6f5438f845b062fa5ab3738/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-10T19:29:08,617 INFO [RS:1;e4ad70981cbb:39803 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e4ad70981cbb%2C39803%2C1733858893985:(num 1733858896462) 2024-12-10T19:29:08,617 DEBUG [RS:1;e4ad70981cbb:39803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:29:08,617 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T19:29:08,617 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T19:29:08,617 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.ChoreService(370): Chore service for: regionserver/e4ad70981cbb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-10T19:29:08,617 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T19:29:08,617 INFO [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:29:08,617 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T19:29:08,617 INFO [regionserver/e4ad70981cbb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T19:29:08,617 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T19:29:08,617 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 15dfd37ce6f5438f845b062fa5ab3738: Waiting for close lock at 1733858948608Running coprocessor pre-close hooks at 1733858948608Disabling compacts and flushes for region at 1733858948608Disabling writes for close at 1733858948608Writing region close event to WAL at 1733858948610 (+2 ms)Running coprocessor post-close hooks at 1733858948617 (+7 ms)Closed at 1733858948617 2024-12-10T19:29:08,617 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T19:29:08,617 DEBUG [RS_CLOSE_REGION-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738. 2024-12-10T19:29:08,617 INFO [RS:1;e4ad70981cbb:39803 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39803 2024-12-10T19:29:08,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e4ad70981cbb,39803,1733858893985 2024-12-10T19:29:08,631 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T19:29:08,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T19:29:08,633 INFO [regionserver/e4ad70981cbb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T19:29:08,635 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/.tmp/info/512523062b1e41c78149011eb3587bc2 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733858917031.15dfd37ce6f5438f845b062fa5ab3738./info:regioninfo/1733858921506/Put/seqid=0 2024-12-10T19:29:08,640 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e4ad70981cbb,39803,1733858893985] 2024-12-10T19:29:08,642 INFO [regionserver/e4ad70981cbb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T19:29:08,647 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e4ad70981cbb,39803,1733858893985 already deleted, retry=false 2024-12-10T19:29:08,648 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e4ad70981cbb,39803,1733858893985 expired; onlineServers=1 2024-12-10T19:29:08,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741938_1118 (size=8243) 2024-12-10T19:29:08,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741938_1118 (size=8243) 2024-12-10T19:29:08,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741938_1118 (size=8243) 2024-12-10T19:29:08,652 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/.tmp/info/512523062b1e41c78149011eb3587bc2 2024-12-10T19:29:08,678 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/.tmp/ns/3b12f4207c0e4e279aaccbe49b65e566 is 43, key is default/ns:d/1733858897309/Put/seqid=0 2024-12-10T19:29:08,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741939_1119 (size=5153) 2024-12-10T19:29:08,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741939_1119 (size=5153) 2024-12-10T19:29:08,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741939_1119 (size=5153) 2024-12-10T19:29:08,685 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/.tmp/ns/3b12f4207c0e4e279aaccbe49b65e566 2024-12-10T19:29:08,703 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/.tmp/table/4dbb8b0da8f04e139e0f62c506f0f2d2 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733858917495/Put/seqid=0 2024-12-10T19:29:08,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741940_1120 (size=5431) 2024-12-10T19:29:08,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741940_1120 (size=5431) 2024-12-10T19:29:08,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741940_1120 (size=5431) 2024-12-10T19:29:08,714 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/.tmp/table/4dbb8b0da8f04e139e0f62c506f0f2d2 2024-12-10T19:29:08,722 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/.tmp/info/512523062b1e41c78149011eb3587bc2 as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/info/512523062b1e41c78149011eb3587bc2 2024-12-10T19:29:08,735 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/info/512523062b1e41c78149011eb3587bc2, entries=18, sequenceid=21, filesize=8.0 K 2024-12-10T19:29:08,736 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/.tmp/ns/3b12f4207c0e4e279aaccbe49b65e566 as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/ns/3b12f4207c0e4e279aaccbe49b65e566 2024-12-10T19:29:08,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T19:29:08,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39803-0x100114e4b300002, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T19:29:08,741 INFO [RS:1;e4ad70981cbb:39803 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T19:29:08,741 INFO [RS:1;e4ad70981cbb:39803 {}] regionserver.HRegionServer(1031): Exiting; stopping=e4ad70981cbb,39803,1733858893985; zookeeper connection closed. 2024-12-10T19:29:08,742 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7fae7145 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7fae7145 2024-12-10T19:29:08,771 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/ns/3b12f4207c0e4e279aaccbe49b65e566, entries=2, sequenceid=21, filesize=5.0 K 2024-12-10T19:29:08,772 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/.tmp/table/4dbb8b0da8f04e139e0f62c506f0f2d2 as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/table/4dbb8b0da8f04e139e0f62c506f0f2d2 2024-12-10T19:29:08,788 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/table/4dbb8b0da8f04e139e0f62c506f0f2d2, entries=2, sequenceid=21, filesize=5.3 K 2024-12-10T19:29:08,789 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 179ms, sequenceid=21, compaction requested=false 2024-12-10T19:29:08,804 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-10T19:29:08,804 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T19:29:08,805 INFO [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T19:29:08,805 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733858948609Running coprocessor pre-close hooks at 1733858948609Disabling compacts and flushes for region at 1733858948609Disabling writes for close at 1733858948610 (+1 ms)Obtaining lock to block concurrent updates at 1733858948610Preparing flush snapshotting stores in 1588230740 at 1733858948610Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733858948610Flushing stores of hbase:meta,,1.1588230740 at 1733858948611 (+1 ms)Flushing 1588230740/info: creating writer at 1733858948612 (+1 ms)Flushing 1588230740/info: appending metadata at 1733858948634 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733858948634Flushing 1588230740/ns: creating writer at 1733858948664 (+30 ms)Flushing 1588230740/ns: appending metadata at 1733858948678 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733858948678Flushing 1588230740/table: creating writer at 1733858948689 (+11 ms)Flushing 1588230740/table: appending metadata at 1733858948703 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733858948703Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63bf0ba: reopening flushed file at 1733858948721 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75e1fa1c: reopening flushed file at 1733858948735 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67664a84: reopening flushed file at 1733858948771 (+36 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 179ms, sequenceid=21, compaction requested=false at 1733858948789 (+18 ms)Writing region close event to WAL at 1733858948800 (+11 ms)Running coprocessor post-close hooks at 1733858948804 (+4 ms)Closed at 1733858948805 (+1 ms) 2024-12-10T19:29:08,805 DEBUG [RS_CLOSE_META-regionserver/e4ad70981cbb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T19:29:08,809 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(976): stopping server e4ad70981cbb,35427,1733858894073; all regions closed. 2024-12-10T19:29:08,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741836_1012 (size=9487) 2024-12-10T19:29:08,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741836_1012 (size=9487) 2024-12-10T19:29:08,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741836_1012 (size=9487) 2024-12-10T19:29:08,815 DEBUG [RS:2;e4ad70981cbb:35427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/oldWALs 2024-12-10T19:29:08,815 INFO [RS:2;e4ad70981cbb:35427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e4ad70981cbb%2C35427%2C1733858894073.meta:.meta(num 1733858897022) 2024-12-10T19:29:08,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741833_1009 (size=1032) 2024-12-10T19:29:08,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741833_1009 (size=1032) 2024-12-10T19:29:08,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741833_1009 (size=1032) 2024-12-10T19:29:08,829 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/WALs/e4ad70981cbb,35427,1733858894073/e4ad70981cbb%2C35427%2C1733858894073.1733858896436 not finished, retry = 0 2024-12-10T19:29:08,932 DEBUG [RS:2;e4ad70981cbb:35427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/oldWALs 2024-12-10T19:29:08,933 INFO [RS:2;e4ad70981cbb:35427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e4ad70981cbb%2C35427%2C1733858894073:(num 1733858896436) 2024-12-10T19:29:08,933 DEBUG [RS:2;e4ad70981cbb:35427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T19:29:08,933 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T19:29:08,933 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T19:29:08,933 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.ChoreService(370): Chore service for: regionserver/e4ad70981cbb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T19:29:08,933 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T19:29:08,933 INFO [regionserver/e4ad70981cbb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T19:29:08,933 INFO [RS:2;e4ad70981cbb:35427 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35427 2024-12-10T19:29:08,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T19:29:08,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e4ad70981cbb,35427,1733858894073 2024-12-10T19:29:08,964 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T19:29:08,973 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e4ad70981cbb,35427,1733858894073] 2024-12-10T19:29:08,981 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e4ad70981cbb,35427,1733858894073 already deleted, retry=false 2024-12-10T19:29:08,981 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e4ad70981cbb,35427,1733858894073 expired; onlineServers=0 2024-12-10T19:29:08,981 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e4ad70981cbb,41471,1733858893036' ***** 2024-12-10T19:29:08,981 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T19:29:08,981 INFO [M:0;e4ad70981cbb:41471 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T19:29:08,981 INFO [M:0;e4ad70981cbb:41471 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T19:29:08,982 DEBUG [M:0;e4ad70981cbb:41471 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T19:29:08,982 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T19:29:08,982 DEBUG [M:0;e4ad70981cbb:41471 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T19:29:08,982 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster-HFileCleaner.large.0-1733858895916 {}] cleaner.HFileCleaner(306): Exit Thread[master/e4ad70981cbb:0:becomeActiveMaster-HFileCleaner.large.0-1733858895916,5,FailOnTimeoutGroup] 2024-12-10T19:29:08,982 DEBUG [master/e4ad70981cbb:0:becomeActiveMaster-HFileCleaner.small.0-1733858895917 {}] cleaner.HFileCleaner(306): Exit Thread[master/e4ad70981cbb:0:becomeActiveMaster-HFileCleaner.small.0-1733858895917,5,FailOnTimeoutGroup] 2024-12-10T19:29:08,982 INFO [M:0;e4ad70981cbb:41471 {}] hbase.ChoreService(370): Chore service for: master/e4ad70981cbb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T19:29:08,982 INFO [M:0;e4ad70981cbb:41471 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T19:29:08,982 DEBUG [M:0;e4ad70981cbb:41471 {}] master.HMaster(1795): Stopping service threads 2024-12-10T19:29:08,982 INFO [M:0;e4ad70981cbb:41471 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T19:29:08,982 INFO [M:0;e4ad70981cbb:41471 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T19:29:08,984 INFO [M:0;e4ad70981cbb:41471 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T19:29:08,984 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T19:29:08,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T19:29:08,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T19:29:08,994 DEBUG [M:0;e4ad70981cbb:41471 {}] zookeeper.ZKUtil(347): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T19:29:08,994 WARN [M:0;e4ad70981cbb:41471 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T19:29:08,995 INFO [M:0;e4ad70981cbb:41471 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/.lastflushedseqids 2024-12-10T19:29:09,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741941_1121 (size=138) 2024-12-10T19:29:09,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741941_1121 (size=138) 2024-12-10T19:29:09,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741941_1121 (size=138) 2024-12-10T19:29:09,038 INFO [M:0;e4ad70981cbb:41471 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T19:29:09,038 INFO [M:0;e4ad70981cbb:41471 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T19:29:09,038 DEBUG [M:0;e4ad70981cbb:41471 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T19:29:09,038 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T19:29:09,038 DEBUG [M:0;e4ad70981cbb:41471 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T19:29:09,038 DEBUG [M:0;e4ad70981cbb:41471 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T19:29:09,038 DEBUG [M:0;e4ad70981cbb:41471 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T19:29:09,038 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-10T19:29:09,057 DEBUG [M:0;e4ad70981cbb:41471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8011afce88e74c5a91603668e4ab6b0f is 82, key is hbase:meta,,1/info:regioninfo/1733858897211/Put/seqid=0 2024-12-10T19:29:09,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T19:29:09,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35427-0x100114e4b300003, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T19:29:09,073 INFO [RS:2;e4ad70981cbb:35427 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T19:29:09,073 INFO [RS:2;e4ad70981cbb:35427 {}] regionserver.HRegionServer(1031): Exiting; stopping=e4ad70981cbb,35427,1733858894073; zookeeper connection closed. 2024-12-10T19:29:09,077 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d25d906 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d25d906 2024-12-10T19:29:09,079 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-10T19:29:09,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741942_1122 (size=5672) 2024-12-10T19:29:09,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741942_1122 (size=5672) 2024-12-10T19:29:09,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741942_1122 (size=5672) 2024-12-10T19:29:09,104 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8011afce88e74c5a91603668e4ab6b0f 2024-12-10T19:29:09,137 DEBUG [M:0;e4ad70981cbb:41471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c40fe3bc6bb848339b633e32d70db67c is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733858917506/Put/seqid=0 2024-12-10T19:29:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741943_1123 (size=7755) 2024-12-10T19:29:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741943_1123 (size=7755) 2024-12-10T19:29:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741943_1123 (size=7755) 2024-12-10T19:29:09,230 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c40fe3bc6bb848339b633e32d70db67c 2024-12-10T19:29:09,239 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c40fe3bc6bb848339b633e32d70db67c 2024-12-10T19:29:09,267 DEBUG [M:0;e4ad70981cbb:41471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/44722662554d4b5cb8653aca019ae309 is 69, key is e4ad70981cbb,35427,1733858894073/rs:state/1733858896099/Put/seqid=0 2024-12-10T19:29:09,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741944_1124 (size=5440) 2024-12-10T19:29:09,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741944_1124 (size=5440) 2024-12-10T19:29:09,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741944_1124 (size=5440) 2024-12-10T19:29:09,346 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/44722662554d4b5cb8653aca019ae309 2024-12-10T19:29:09,363 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 44722662554d4b5cb8653aca019ae309 2024-12-10T19:29:09,364 DEBUG [M:0;e4ad70981cbb:41471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8011afce88e74c5a91603668e4ab6b0f as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8011afce88e74c5a91603668e4ab6b0f 2024-12-10T19:29:09,375 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8011afce88e74c5a91603668e4ab6b0f, entries=8, sequenceid=168, filesize=5.5 K 2024-12-10T19:29:09,376 DEBUG [M:0;e4ad70981cbb:41471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c40fe3bc6bb848339b633e32d70db67c as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c40fe3bc6bb848339b633e32d70db67c 2024-12-10T19:29:09,384 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c40fe3bc6bb848339b633e32d70db67c 2024-12-10T19:29:09,385 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c40fe3bc6bb848339b633e32d70db67c, entries=17, sequenceid=168, filesize=7.6 K 2024-12-10T19:29:09,389 DEBUG [M:0;e4ad70981cbb:41471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/44722662554d4b5cb8653aca019ae309 as hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/44722662554d4b5cb8653aca019ae309 2024-12-10T19:29:09,408 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 44722662554d4b5cb8653aca019ae309 2024-12-10T19:29:09,408 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46813/user/jenkins/test-data/3133bad7-616d-ca5c-d274-520caf5c8d36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/44722662554d4b5cb8653aca019ae309, entries=3, sequenceid=168, filesize=5.3 K 2024-12-10T19:29:09,421 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 383ms, sequenceid=168, compaction requested=false 2024-12-10T19:29:09,437 INFO [M:0;e4ad70981cbb:41471 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T19:29:09,437 DEBUG [M:0;e4ad70981cbb:41471 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733858949038Disabling compacts and flushes for region at 1733858949038Disabling writes for close at 1733858949038Obtaining lock to block concurrent updates at 1733858949038Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733858949038Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733858949038Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733858949039 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733858949039Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733858949056 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733858949057 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733858949115 (+58 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733858949136 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733858949136Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733858949239 (+103 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733858949266 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733858949266Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fb33f20: reopening flushed file at 1733858949363 (+97 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6970c339: reopening flushed file at 1733858949375 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7067c8fa: reopening flushed file at 1733858949385 (+10 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 383ms, sequenceid=168, compaction requested=false at 1733858949421 (+36 ms)Writing region close event to WAL at 1733858949437 (+16 ms)Closed at 1733858949437 2024-12-10T19:29:09,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43797 is added to blk_1073741830_1006 (size=83606) 2024-12-10T19:29:09,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35009 is added to blk_1073741830_1006 (size=83606) 2024-12-10T19:29:09,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39621 is added to blk_1073741830_1006 (size=83606) 2024-12-10T19:29:09,464 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T19:29:09,464 INFO [M:0;e4ad70981cbb:41471 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T19:29:09,465 INFO [M:0;e4ad70981cbb:41471 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41471 2024-12-10T19:29:09,466 INFO [M:0;e4ad70981cbb:41471 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T19:29:09,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T19:29:09,606 INFO [M:0;e4ad70981cbb:41471 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T19:29:09,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41471-0x100114e4b300000, quorum=127.0.0.1:53100, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T19:29:09,614 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858923874 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858923874 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733858922321/wal.1733858923874 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T19:29:09,615 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733858911371/wal.1733858912150 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:29:09,615 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733858941666/wal.1733858942916 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:29:09,618 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858934816 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858934816 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733858924606/wal.1733858934816 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T19:29:09,619 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733858900534/wal.1733858900686 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:29:09,621 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858936377 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858936377 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733858935185/wal.1733858936377 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T19:29:09,622 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733858936750/wal.1733858936925 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T19:29:09,625 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal.1733858900227 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal.1733858900227 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733858899137/wal.1733858900227 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T19:29:09,628 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal.1733858898543 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal.1733858898543 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733858897727/wal.1733858898543 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1225853664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T19:29:09,639 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T19:29:09,642 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T19:29:09,642 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T19:29:09,642 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T19:29:09,642 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.log.dir/,STOPPED} 2024-12-10T19:29:09,657 WARN [BP-1608359495-172.17.0.3-1733858888650 heartbeating to localhost/127.0.0.1:46813 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T19:29:09,657 WARN [BP-1608359495-172.17.0.3-1733858888650 heartbeating to localhost/127.0.0.1:46813 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1608359495-172.17.0.3-1733858888650 (Datanode Uuid d842e86a-de0e-42d5-8d40-f4a25915d3a3) service to localhost/127.0.0.1:46813 2024-12-10T19:29:09,658 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T19:29:09,659 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T19:29:09,660 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data5/current/BP-1608359495-172.17.0.3-1733858888650 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T19:29:09,660 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data6/current/BP-1608359495-172.17.0.3-1733858888650 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T19:29:09,661 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T19:29:09,673 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T19:29:09,676 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T19:29:09,676 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T19:29:09,676 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T19:29:09,676 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.log.dir/,STOPPED} 2024-12-10T19:29:09,686 WARN [BP-1608359495-172.17.0.3-1733858888650 heartbeating to localhost/127.0.0.1:46813 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T19:29:09,686 WARN [BP-1608359495-172.17.0.3-1733858888650 heartbeating to localhost/127.0.0.1:46813 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1608359495-172.17.0.3-1733858888650 (Datanode Uuid 10ffcbc2-356c-4f0d-b302-01e6f5dbcc2b) service to localhost/127.0.0.1:46813 2024-12-10T19:29:09,687 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data3/current/BP-1608359495-172.17.0.3-1733858888650 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T19:29:09,687 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data4/current/BP-1608359495-172.17.0.3-1733858888650 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T19:29:09,687 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T19:29:09,688 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T19:29:09,688 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T19:29:09,702 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T19:29:09,702 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T19:29:09,702 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T19:29:09,702 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T19:29:09,702 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.log.dir/,STOPPED} 2024-12-10T19:29:09,709 WARN [BP-1608359495-172.17.0.3-1733858888650 heartbeating to localhost/127.0.0.1:46813 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T19:29:09,709 WARN [BP-1608359495-172.17.0.3-1733858888650 heartbeating to localhost/127.0.0.1:46813 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1608359495-172.17.0.3-1733858888650 (Datanode Uuid 134c8558-deb3-477d-8cc5-a583a3334372) service to localhost/127.0.0.1:46813 2024-12-10T19:29:09,710 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data1/current/BP-1608359495-172.17.0.3-1733858888650 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T19:29:09,710 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T19:29:09,712 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T19:29:09,712 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/cluster_adeca968-4b0c-3755-0032-870d74d499bf/data/data2/current/BP-1608359495-172.17.0.3-1733858888650 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T19:29:09,712 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T19:29:09,725 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T19:29:09,726 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T19:29:09,726 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T19:29:09,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T19:29:09,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/467538ed-e85b-d96b-4804-00d810f5a603/hadoop.log.dir/,STOPPED} 2024-12-10T19:29:09,751 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T19:29:09,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down